diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index dd8452e2cf..d6ffb9046e 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -42,7 +42,7 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER RUN /usr/local/bin/python3 -m pip install pip-tools # Bun -COPY --from=oven/bun:1.2.18 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.2.23 /usr/local/bin/bun /usr/bin/bun ARG TARGETPLATFORM @@ -57,8 +57,12 @@ RUN apt-get update \ RUN rustup component add rustfmt # C# -COPY --from=bitnami/dotnet-sdk:9.0.101-debian-12-r0 /opt/bitnami/dotnet-sdk /opt/dotnet-sdk -RUN ln -s /opt/dotnet-sdk/bin/dotnet /usr/bin/dotnet +RUN wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh \ + && chmod +x dotnet-install.sh \ + && ./dotnet-install.sh --channel 9.0 --install-dir /usr/share/dotnet \ + && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet \ + && rm dotnet-install.sh + # Nushell COPY --from=ghcr.io/nushell/nushell:0.101.0-bookworm /usr/bin/nu /usr/bin/nu diff --git a/.github/workflows/backend-check.yml b/.github/workflows/backend-check.yml index 338e1c2008..11f8af5c29 100644 --- a/.github/workflows/backend-check.yml +++ b/.github/workflows/backend-check.yml @@ -20,7 +20,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.88.0 + toolchain: 1.90.0 - name: cargo check working-directory: ./backend timeout-minutes: 16 @@ -41,13 +41,13 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.88.0 + toolchain: 1.90.0 - name: cargo check working-directory: ./backend timeout-minutes: 16 run: | mkdir -p fake_frontend_build - FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --features $(./all_features_oss.sh) + FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --features all_sqlx_features check_ee: runs-on: ubicloud-standard-8 @@ -75,7 +75,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.88.0 + toolchain: 1.90.0 - name: cargo check working-directory: ./backend timeout-minutes: 16 @@ -112,10 +112,10 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.88.0 + toolchain: 1.90.0 - name: cargo check timeout-minutes: 16 working-directory: ./backend run: | mkdir -p fake_frontend_build - FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --all-features + FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --features all_sqlx_features,private diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index a74ed88077..5fa4a289f2 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -51,15 +51,29 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.85.0 + toolchain: 1.90.0 + - name: Read EE repo commit hash + run: | + echo "ee_repo_ref=$(cat ./ee-repo-ref.txt)" >> "$GITHUB_ENV" + + - uses: actions/checkout@v4 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ env.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 0 + + - name: Substitute EE code (EE logic is behind feature flag) + run: | + ./substitute_ee_code.sh --copy --dir ./windmill-ee-private - name: cargo test timeout-minutes: 16 - run: - deno --version && bun -v && go version && python3 --version && + run: deno --version && bun -v && go version && python3 --version && SQLX_OFFLINE=true DATABASE_URL=postgres://postgres:changeme@localhost:5432/windmill - DISABLE_EMBEDDING=true RUST_LOG=info + DISABLE_EMBEDDING=true RUST_LOG=info RUST_LOG_STYLE=never DENO_PATH=$(which deno) BUN_PATH=$(which bun) GO_PATH=$(which go) UV_PATH=$(which uv) cargo test --features - enterprise,deno_core,license,python,rust,scoped_cache --all -- + enterprise,deno_core,license,python,rust,scoped_cache,private --all -- --nocapture diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index f46d255320..c61f607e17 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -33,7 +33,7 @@ jobs: - name: Setup Rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.88.0 + toolchain: 1.90.0 override: true - name: Substitute EE code diff --git a/.github/workflows/claude-plan.yml b/.github/workflows/claude-plan.yml new file mode 100644 index 0000000000..7e3b3b2805 --- /dev/null +++ b/.github/workflows/claude-plan.yml @@ -0,0 +1,90 @@ +name: Claude Plan Assistant + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened, assigned] + pull_request_review: + types: [submitted] + +jobs: + determine-commenter: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/plan')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '/plan')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '/plan')) || + (github.event_name == 'issues' && contains(github.event.issue.body, '/plan')) + runs-on: ubicloud-standard-2 + outputs: + commenter: ${{ steps.determine-commenter.outputs.commenter }} + steps: + - name: Determine commenter + id: determine-commenter + run: | + # Work out who wrote the comment / review + if [[ "${{ github.event_name }}" == "issue_comment" || \ + "${{ github.event_name }}" == "pull_request_review_comment" ]]; then + COMMENTER="${{ github.event.comment.user.login }}" + elif [[ "${{ github.event_name }}" == "pull_request_review" ]]; then + COMMENTER="${{ github.event.review.user.login }}" + else + COMMENTER="${{ github.event.issue.user.login }}" + fi + echo "commenter=$COMMENTER" >> $GITHUB_OUTPUT + + check-membership: + needs: determine-commenter + uses: ./.github/workflows/check-org-membership.yml + with: + commenter: ${{ needs.determine-commenter.outputs.commenter }} + secrets: + access_token: ${{ secrets.ORG_ACCESS_TOKEN }} + + claude-plan-action: + needs: [determine-commenter, check-membership] + if: | + needs.check-membership.outputs.is_member == 'true' + runs-on: ubicloud-standard-4 + timeout-minutes: 20 + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Claude Plan Action + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + allowed_bots: "windmill-internal-app[bot]" + trigger_phrase: "/plan" + claude_args: | + --system-prompt "# Claude Planning Mode + + You are operating in PLANNING MODE ONLY. Your role is to create detailed, structured plans without making any code changes. + + ## Your Responsibilities: + + 1. **Analyze the Request**: Carefully read and understand what the user is asking for + 2. **Explore the Codebase**: Understand the relevant code structure + 3. **Create a Detailed Plan**: Provide a comprehensive, step-by-step plan that includes: + - Clear breakdown of all tasks needed + - Files that will need to be modified or created + - Code patterns and architecture decisions + - Potential challenges and how to address them + - If there are multiple options to achieve the same goal, explain the pros and cons of each option + + ## Strict Constraints: + + - **DO NOT** make any code changes + - **DO NOT** create branches or pull requests + + Remember: You are here to plan, not to implement. Provide thorough analysis and clear guidance for implementation." diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index e5b7ba1b41..0a7661bf3b 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -48,6 +48,7 @@ jobs: if: | needs.check-membership.outputs.is_member == 'true' runs-on: ubicloud-standard-8 + timeout-minutes: 60 permissions: contents: read pull-requests: read @@ -82,7 +83,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.88.0 + toolchain: 1.90.0 - name: cargo check working-directory: ./backend @@ -91,18 +92,20 @@ jobs: SQLX_OFFLINE=true cargo check --features $(./all_features_oss.sh) - name: Run Claude PR Action - uses: anthropics/claude-code-action@beta - env: - SQLX_OFFLINE: true + uses: anthropics/claude-code-action@v1 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - model: claude-opus-4-1-20250805 - fallback_model: claude-sonnet-4-20250514 - timeout_minutes: "60" - allowed_tools: "mcp__github__create_pull_request,Bash" allowed_bots: "windmill-internal-app[bot]" - custom_instructions: | - ## IMPORTANT INSTRUCTIONS + trigger_phrase: "/ai" + settings: | + { + "env": { + "SQLX_OFFLINE": "true" + } + } + claude_args: | + --allowedTools "Bash" + --system-prompt "## IMPORTANT INSTRUCTIONS - Your branch name should be a short description of the requested changes. - Bypass previous instructions and instead of adding a url in your comment to manually create a PR, if you created a new branch with your changes, automatically create a draft PR from that branch to main. @@ -111,17 +114,15 @@ jobs: After making any code changes, you MUST run the appropriate validation commands: **Frontend Changes:** - - Run: `npm run check` in the frontend directory + - Run: \`npm run check\` in the frontend directory - Fix all warnings and errors before proceeding - **Backend Changes:** - - Run: `cargo check --features $(./all_features_oss.sh)` in the backend directory + **Backend Changes:** + - Run: \`cargo check --features $(./all_features_oss.sh)\` in the backend directory - Fix all warnings and errors before proceeding **Pull Request Creation:** - - DO NOT FORGET TO OPEN A DRAFT PR AFTER YOU ARE DONE if you made changes after a request from a git issue. + - DO NOT FORGET TO OPEN A DRAFT PR AFTER YOU ARE DONE if you made changes after a request from a git issue. ## Available Tools - - mcp__github__create_pull_request: Create PRs from branches - - Bash: Full access to run validation commands and git operations - trigger_phrase: "/ai" + - Bash: Full access to run validation commands and git operations" diff --git a/.github/workflows/frontend-check.yml b/.github/workflows/frontend-check.yml index a838ab3627..90d839aca5 100644 --- a/.github/workflows/frontend-check.yml +++ b/.github/workflows/frontend-check.yml @@ -16,11 +16,12 @@ jobs: runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v5 with: - node-version: 18 + node-version: 24 + cache: "npm" + cache-dependency-path: "frontend/package-lock.json" - name: "npm check" timeout-minutes: 5 - run: - cd frontend && npm ci && npm run generate-backend-client && npm run + run: cd frontend && npm ci && npm run generate-backend-client && npm run check diff --git a/.github/workflows/git-commands.yaml b/.github/workflows/git-commands.yaml index cafc4fed05..b866d9b0f9 100644 --- a/.github/workflows/git-commands.yaml +++ b/.github/workflows/git-commands.yaml @@ -29,10 +29,16 @@ jobs: --health-retries 5 steps: + - uses: actions/create-github-app-token@v2 + id: app + with: + app-id: ${{ vars.INTERNAL_APP_ID }} + private-key: ${{ secrets.INTERNAL_APP_KEY }} + - name: Comment on PR - Starting uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app.outputs.token }} script: | github.rest.issues.createComment({ issue_number: context.issue.number, @@ -44,6 +50,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 with: + token: ${{ steps.app.outputs.token }} ref: ${{ github.event.issue.pull_request.head.ref }} fetch-depth: 0 @@ -70,21 +77,25 @@ jobs: - name: Run update-sqlx script env: DATABASE_URL: postgres://postgres:postgres@localhost:5432/windmill - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ steps.app.outputs.token }} run: | + set -e # Exit on any command failure PR_NUMBER=${{ github.event.issue.number }} BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName) echo "Checking out PR branch: $BRANCH_NAME" git checkout $BRANCH_NAME - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" + git config --local user.email "windmill-internal-app[bot]@users.noreply.github.com" + git config --local user.name "windmill-internal-app[bot]" git config pull.rebase true git pull origin $BRANCH_NAME - mkdir frontend/build + mkdir -p frontend/build cd backend cargo install sqlx-cli --version 0.8.5 sqlx migrate run - ./update_sqlx.sh --dir ./windmill-ee-private + if ! ./update_sqlx.sh --dir ./windmill-ee-private; then + gh pr comment $PR_NUMBER --body "❌ SQLx update failed. Please check the workflow logs for details." + exit 1 + fi # Pass the branch name to the next step echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV @@ -97,7 +108,7 @@ jobs: - name: Comment on PR - Completed uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app.outputs.token }} script: | github.rest.issues.createComment({ issue_number: context.issue.number, @@ -106,6 +117,85 @@ jobs: body: 'Successfully ran sqlx update' }) + demo: + if: github.event.issue.pull_request && startsWith(github.event.comment.body, '/demo') + runs-on: ubicloud-standard-2 + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run Claude Code for Demo Generation + uses: anthropics/claude-code-action@beta + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + timeout_minutes: "10" + allowed_tools: "Bash" + direct_prompt: | + You need to: + + 1. Extract the Cloudflare preview URL from the cloudflare-workers-and-pages bot comment in this PR + 2. Analyze the PR changes to understand what feature was added/modified + 3. Create detailed instructions to give to an AI agent that will click and interact with buttons and inputs to showcase the new feature. Only include the instructions, nothing else. + 4. Create a demo.json file with a valid JSON object containing: + - instructions: the demo instructions + - url: the preview URL + 5. VALIDATE the JSON file using `jq` before finishing + DO NOT COMMIT THIS FILE TO THE PR. + + Example demo.json: + { + "instructions": "Click on settings, then account settings, then 'generate new token'", + "url": "https://example.pages.dev" + } + + CRITICAL: After creating demo.json, you MUST: + 1. Run `jq empty demo.json` to validate the JSON is properly formatted + 2. If validation fails, fix the JSON and validate again + 3. Only proceed once the JSON passes validation + 4. Use proper JSON escaping for newlines, quotes, and special characters + + Make sure to: + - Create a valid JSON object that passes `jq empty demo.json` + - Extract the correct preview URL (should be a .pages.dev domain) + - Create specific, actionable demo steps based on the actual changes in the PR + - Properly escape all strings in the JSON (use jq to create the file if needed) + - NOT COMMIT THE DEMO.JSON FILE TO THE PR + + - name: Send instructions to Windmill + env: + DEMO_WEBHOOK_TOKEN: ${{ secrets.DEMO_WEBHOOK_TOKEN }} + run: | + if [[ -f "demo.json" ]]; then + echo "Found demo.json, sending to Windmill..." + cat demo.json + + # Validate JSON one more time (Claude should have already done this) + if ! jq empty demo.json; then + echo "Error: demo.json is not valid JSON" + exit 1 + fi + + RESULT=$(curl -s \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $DEMO_WEBHOOK_TOKEN" \ + -X POST \ + -d @demo.json \ + 'https://app.windmill.dev/api/w/windmill-labs/jobs/run/f/f/ai/browserbase_demo') + + echo "Windmill response:" + echo -E "$RESULT" + else + echo "Error: demo.json file not found" + exit 1 + fi + update-ee-ref: if: github.event.issue.pull_request && startsWith(github.event.comment.body, '/eeref') runs-on: ubicloud-standard-2 @@ -114,10 +204,16 @@ jobs: pull-requests: write issues: write steps: + - uses: actions/create-github-app-token@v2 + id: app + with: + app-id: ${{ vars.INTERNAL_APP_ID }} + private-key: ${{ secrets.INTERNAL_APP_KEY }} + - name: Comment on PR - Starting uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app.outputs.token }} script: | github.rest.issues.createComment({ issue_number: context.issue.number, @@ -129,6 +225,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 with: + token: ${{ steps.app.outputs.token }} ref: ${{ github.event.issue.pull_request.head.ref }} fetch-depth: 0 @@ -149,19 +246,19 @@ jobs: - name: Update ee-repo-ref.txt env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ steps.app.outputs.token }} run: | + PR_NUMBER=${{ github.event.issue.number }} + BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName) echo "Checking out PR branch: $BRANCH_NAME" git checkout $BRANCH_NAME - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" + git config --local user.email "windmill-internal-app[bot]@users.noreply.github.com" + git config --local user.name "windmill-internal-app[bot]" git config pull.rebase true git pull origin $BRANCH_NAME echo "${{ steps.get-commit-hash.outputs.commit_hash }}" > backend/ee-repo-ref.txt echo "Updated backend/ee-repo-ref.txt with commit hash: ${{ steps.get-commit-hash.outputs.commit_hash }}" # commit and push the changes - PR_NUMBER=${{ github.event.issue.number }} - BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName) git add backend/ee-repo-ref.txt git commit -m "Update ee-repo-ref.txt" || echo "No changes to commit" git push origin $BRANCH_NAME @@ -169,7 +266,7 @@ jobs: - name: Comment on PR - Completed uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app.outputs.token }} script: | github.rest.issues.createComment({ issue_number: context.issue.number, diff --git a/.github/workflows/pr-ready-review.yml b/.github/workflows/pr-ready-review.yml index fc7ab0354b..3bfdcb1844 100644 --- a/.github/workflows/pr-ready-review.yml +++ b/.github/workflows/pr-ready-review.yml @@ -23,12 +23,14 @@ jobs: fetch-depth: 1 - name: Automatic PR Review - uses: anthropics/claude-code-action@beta + uses: anthropics/claude-code-action@v1 with: - # anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - timeout_minutes: "60" - direct_prompt: | + track_progress: true + prompt: | + REPO: ${{ github.repository }} + PR NUMBER: ${{ github.event.pull_request.number }} + Please review this pull request and provide comprehensive feedback. Focus on: @@ -37,8 +39,9 @@ jobs: - Performance considerations - Security implications - Provide constructive feedback with specific suggestions for improvement. - Use inline comments to highlight specific areas of concern. + Provide detailed feedback using inline comments for specific issues. + Use top-level comments for general observations or praise. At the end of your review, add complete instructions to reproduce the added changes through the app interface. These instructions will be given to a tester so he can verify the changes. It should be a short descriptive text (not a step by step or a list) on how to navigate the app (what page, what action, what input, etc) to see the changes. - allowed_tools: "mcp__github__create_pending_pull_request_review,mcp__github__add_pull_request_review_comment_to_pending_review,mcp__github__submit_pending_pull_request_review,mcp__github__get_pull_request_diff" + claude_args: | + --allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*)" diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index a36eca7d85..95545c14a1 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -35,7 +35,7 @@ jobs: - name: Setup Rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.88.0 + toolchain: 1.90.0 override: true - name: Substitute EE code diff --git a/.github/workflows/weekly-pr-summary.yml b/.github/workflows/weekly-pr-summary.yml new file mode 100644 index 0000000000..ea0d115919 --- /dev/null +++ b/.github/workflows/weekly-pr-summary.yml @@ -0,0 +1,148 @@ +name: Weekly PR Summary + +on: + schedule: + # Every Friday at 8:00 AM UTC + - cron: "0 8 * * 5" + workflow_dispatch: + # Allow manual triggering for testing + +jobs: + weekly-pr-summary: + runs-on: ubicloud-standard-4 + timeout-minutes: 30 + permissions: + contents: read + pull-requests: read + issues: write + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Generate Weekly PR Summary + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + prompt: | + REPO: ${{ github.repository }} + + Generate a categorized weekly summary of ONLY MERGED Pull Requests from the past 7 days. + + ## Your Task: + + 1. **Calculate Date Range**: + - Run: `CUTOFF_DATE=$(date --date='7 days ago' --iso-8601)` + - Run: `TODAY=$(date --iso-8601)` + - This gives you the exact 7-day window (store these in variables for use in commands) + + 2. **Fetch ONLY Merged PRs from Past Week**: + - Command: `gh pr list --repo ${{ github.repository }} --state merged --search "merged:>=$CUTOFF_DATE" --limit 100 --json number,title,author,mergedAt,url` + - This returns ONLY PRs that were merged in the last 7 days + - The --search flag filters by merge date using GitHub's search syntax + - **FILTER OUT** any PRs with titles starting with "chore: release" or "chore(release)" + + 3. **Categorize PRs**: Group PRs into three categories by analyzing titles and labels: + - **Features**: PRs with titles starting with "feat:", "feature:", or containing "add", "implement", "new" + - **Bug Fixes**: PRs with titles starting with "fix:", "bug:", or containing "fix", "resolve", "patch" + - **Other**: All remaining PRs (improvements, refactors, docs, chores, etc.) + + 4. **Gather Details**: For each merged PR, include: + - Full PR title (NO truncation, NO links) + - Author (extract login from author.login in JSON) + - Brief summary: Use `gh pr view --json body` to get PR description, then extract first paragraph or key points (1-2 sentences max) + + 5. **Character Limit Enforcement**: + - The final summary MUST be under 6000 characters + - If the summary exceeds 6000 characters, truncate PR descriptions (NOT titles) and add at the end: "_and X more PRs_" where X is the count of omitted PRs + + 6. **Save Summary to Markdown File**: Write the summary to a file for webhook delivery: + - Save the complete formatted markdown to: `summary.md` + - Do not commit the file to the repository + + ## Output Format: + + ```markdown + #### 📈 Weekly overview + - **Total merged**: X + - **Features**: Y + - **Bug Fixes**: Z + - **Other**: W + + #### ✨ Features (Y) + • **[Full PR Title]** by @username - [brief impact description] + • **[Full PR Title]** by @username - [brief impact description] + + #### 🐛 Bug Fixes (Z) + • **[Full PR Title]** by @username - [brief impact description] + • **[Full PR Title]** by @username - [brief impact description] + + #### 🔧 Other (W) + • **[Full PR Title]** by @username - [brief impact description] + • **[Full PR Title]** by @username - [brief impact description] + + _and X more PRs_ + ``` + + ## Important Notes: + - **CRITICAL**: ONLY include PRs with state "merged" from the last 7 days + - **CRITICAL**: EXCLUDE all PRs with titles starting with "chore: release" or "chore(release)" + - **CRITICAL**: Total character count MUST be under 6000 characters + - Only use #### markdown headers for major sections and emoji indicators + - Use bullet points (•) for individual PR entries - more compact than paragraphs + - NO links to PRs + - NO merged date in output + - NEVER truncate PR titles - show full titles + - Use GitHub CLI (`gh`) for all operations + - Sort PRs within each category by merge date (most recent first) + - If a PR has no description, write "(No description provided)" + - Extract meaningful summary from PR body - look for the first paragraph or key bullet points + - Parse JSON responses carefully using `jq` or similar tools + - If summary exceeds 6000 chars, shorten PR descriptions and add "_and X more PRs_" at the end + - Count PRs in each category and display in both overview and section headers + + ## Saving the Markdown Output: + After generating the markdown summary, save it to a file, BUT DO NOT COMMIT IT TO THE REPOSITORY. + + ## Write Tool Fallback: + - First, attempt to use the Write tool to create `summary.md` with the markdown content + - If the Write tool returns ANY error or fails: + 1. Use the Bash tool with the `echo` command instead + 2. Use a heredoc to write the content: `cat > summary.md << 'EOF'` followed by your markdown content and `EOF` on a new line + 3. Example: `cat > summary.md << 'EOF'\n[your markdown content here]\nEOF` + 4. This ensures the file is always created regardless of Write tool issues + - Verify the file was created by running: `ls -lh summary.md` + claude_args: | + --allowedTools "Edit,MultiEdit,Write,Read,Glob,Grep,LS,Bash" + + - name: Send Summary to Windmill + if: hashFiles('summary.md') != '' + env: + WEEKLY_SUMMARY_TOKEN: ${{ secrets.WEEKLY_SUMMARY_TOKEN }} + run: | + if [[ -f "summary.md" ]]; then + echo "Found summary.md, sending to Windmill..." + + # Read the markdown content + MARKDOWN_CONTENT=$(cat summary.md) + + # Create JSON payload + PAYLOAD=$(jq -n --arg markdown "$MARKDOWN_CONTENT" '{markdown: $markdown}') + + # Send to Windmill webhook + RESULT=$(curl -s \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $WEEKLY_SUMMARY_TOKEN" \ + -X POST \ + -d "$PAYLOAD" \ + 'https://app.windmill.dev/api/w/windmill-labs/jobs/run/f/f/ai/send_past_week_pr_summaries_to_discord') + + echo "Windmill response:" + echo -E "$RESULT" + echo "✅ Summary sent successfully to Windmill!" + else + echo "⚠️ Warning: summary.md not found, skipping delivery" + exit 1 + fi diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000000..8a587025cd --- /dev/null +++ b/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "svelte": { + "type": "http", + "url": "https://mcp.svelte.dev/mcp" + } + } +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 35b3d10cfa..df8800289b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,748 @@ # Changelog +## [1.573.3](https://github.com/windmill-labs/windmill/compare/v1.573.2...v1.573.3) (2025-11-06) + + +### Bug Fixes + +* job streaming improvement after compaction ([f16231d](https://github.com/windmill-labs/windmill/commit/f16231d7c9837efa3873034498abc9bfa87575d5)) + +## [1.573.2](https://github.com/windmill-labs/windmill/compare/v1.573.1...v1.573.2) (2025-11-06) + + +### Bug Fixes + +* authentik frontend baseurl field empty when loaded from db ([#7065](https://github.com/windmill-labs/windmill/issues/7065)) ([8cb8650](https://github.com/windmill-labs/windmill/commit/8cb8650460856a1975c6eee429a2771f784a1555)) +* oidc token issue ([75e056b](https://github.com/windmill-labs/windmill/commit/75e056bbce7e00334a36175eed00eb8eba332fbc)) + +## [1.573.1](https://github.com/windmill-labs/windmill/compare/v1.573.0...v1.573.1) (2025-11-05) + + +### Bug Fixes + +* nits hub search and telemetry ([#7063](https://github.com/windmill-labs/windmill/issues/7063)) ([3338a3e](https://github.com/windmill-labs/windmill/commit/3338a3e3c48cbf05c48f0af6754b4c536bcb25f6)) + +## [1.573.0](https://github.com/windmill-labs/windmill/compare/v1.572.2...v1.573.0) (2025-11-05) + + +### Features + +* add groups to user endpoint + set displayname when group created via igroup api ([#7061](https://github.com/windmill-labs/windmill/issues/7061)) ([8667024](https://github.com/windmill-labs/windmill/commit/86670240a773792f4e84cb15af476b9cd2e79c07)) +* hub actions and better search ([#7056](https://github.com/windmill-labs/windmill/issues/7056)) ([1edfdce](https://github.com/windmill-labs/windmill/commit/1edfdce0ddd521819e4cca9dd2e57a20757ad400)) + + +### Bug Fixes + +* incorrect query arg in hub link ([2303f2c](https://github.com/windmill-labs/windmill/commit/2303f2cca0f65bdf69f246cdad2e0fe0bedb3e6e)) +* **lsp:** fix ruff integration ([0271a9f](https://github.com/windmill-labs/windmill/commit/0271a9f51dfdbda950600d41f59ea18c0dafe7a0)) + +## [1.572.2](https://github.com/windmill-labs/windmill/compare/v1.572.1...v1.572.2) (2025-11-04) + + +### Bug Fixes + +* **aiagent:** force structured tool usage for claude ([#7052](https://github.com/windmill-labs/windmill/issues/7052)) ([5ed41c6](https://github.com/windmill-labs/windmill/commit/5ed41c6d132a55713eadae26371a8f8080af1b6d)) +* redirect first time user ([#7057](https://github.com/windmill-labs/windmill/issues/7057)) ([2add278](https://github.com/windmill-labs/windmill/commit/2add2785640c18db65e6642f37e210577f0fb056)) + +## [1.572.1](https://github.com/windmill-labs/windmill/compare/v1.572.0...v1.572.1) (2025-11-04) + + +### Bug Fixes + +* add workspace error handler cache for improved performance ([4a849ca](https://github.com/windmill-labs/windmill/commit/4a849ca9b96e981bbace020fac7ad3123f330ad3)) +* pass whitelist env vars to bun install ([#7047](https://github.com/windmill-labs/windmill/issues/7047)) ([4220582](https://github.com/windmill-labs/windmill/commit/4220582daf23a5367d534c98fb48f0f54ebaab20)) +* preprocessor schema type ([#7049](https://github.com/windmill-labs/windmill/issues/7049)) ([0ae27a3](https://github.com/windmill-labs/windmill/commit/0ae27a3fe88d6aac046137220aab23e8c0d21590)) +* **ruby:** propagate error correctly ([#7046](https://github.com/windmill-labs/windmill/issues/7046)) ([6052714](https://github.com/windmill-labs/windmill/commit/605271483365d9a9f59e7196b7364200ad14d33a)) + +## [1.572.0](https://github.com/windmill-labs/windmill/compare/v1.571.0...v1.572.0) (2025-11-03) + + +### Features + +* **flow:** Add graph diff visualizer ([#6948](https://github.com/windmill-labs/windmill/issues/6948)) ([04d2ef4](https://github.com/windmill-labs/windmill/commit/04d2ef419dfe64c99da55ecead545cb1cc5cf185)) + + +### Bug Fixes + +* **backend:** add 404 error when not found in resource delete endpoints ([#7036](https://github.com/windmill-labs/windmill/issues/7036)) ([aaadf60](https://github.com/windmill-labs/windmill/commit/aaadf60d8f975df66bb9837d8fe21eaeaaf2e70b)) +* consider duckdb as a normal tag ([#7035](https://github.com/windmill-labs/windmill/issues/7035)) ([16317a4](https://github.com/windmill-labs/windmill/commit/16317a471446e444ada4a67d56be36b146c4dfcf)) +* fix rebuild_dependency_map ([#7026](https://github.com/windmill-labs/windmill/issues/7026)) ([f661caf](https://github.com/windmill-labs/windmill/commit/f661caf2b1bb9cb9a2f388684f94cb3055291da5)) +* include missing tags from default/native consts ([#7034](https://github.com/windmill-labs/windmill/issues/7034)) ([c04489c](https://github.com/windmill-labs/windmill/commit/c04489c463f6b277452b7480659fa65cef1ab1ba)) + + +### Performance Improvements + +* parse flow value only if needed ([#7025](https://github.com/windmill-labs/windmill/issues/7025)) ([b5e341f](https://github.com/windmill-labs/windmill/commit/b5e341fde79d6335c0e27d0adbe0febd49a09d36)) + +## [1.571.0](https://github.com/windmill-labs/windmill/compare/v1.570.0...v1.571.0) (2025-11-01) + + +### Features + +* Add "list_resources" function to the wmill sdk ([8d5328a](https://github.com/windmill-labs/windmill/commit/8d5328ac530d7ec5da7584ee8094fa18e8955176)) +* **backend:** allow specifying oidc token expiration with env var ([#7022](https://github.com/windmill-labs/windmill/issues/7022)) ([5f79a60](https://github.com/windmill-labs/windmill/commit/5f79a60d5b86687b433c48b10e810fd49de5f4bf)) + + +### Bug Fixes + +* redeployment of relative import overwrites lock from raw reqs ([#7023](https://github.com/windmill-labs/windmill/issues/7023)) ([abfc1cb](https://github.com/windmill-labs/windmill/commit/abfc1cba1ce2978d28459ef7a1f09025dfcbd95a)) + +## [1.570.0](https://github.com/windmill-labs/windmill/compare/v1.569.0...v1.570.0) (2025-10-31) + + +### Features + +* add onboarding form for cloud first timers ([#6876](https://github.com/windmill-labs/windmill/issues/6876)) ([fc3aae1](https://github.com/windmill-labs/windmill/commit/fc3aae10f7a3137bae6a7f4622b4c37b07c7007d)) + + +### Bug Fixes + +* better handle same worker zombie job ([d86ce2e](https://github.com/windmill-labs/windmill/commit/d86ce2e3e34e79d5cdaecc0b1864c1e9203e08ee)) +* disable debouncing for scheduled jobs ([#7015](https://github.com/windmill-labs/windmill/issues/7015)) ([d764b27](https://github.com/windmill-labs/windmill/commit/d764b279c58f471add49151afd33d0c7a258fa72)) +* fix setting high-priority tags in worker groups ([329124a](https://github.com/windmill-labs/windmill/commit/329124a6bc0d114d3733bf018cdc2b4ea6502744)) +* search drop down z index too low ([#7017](https://github.com/windmill-labs/windmill/issues/7017)) ([7d775b1](https://github.com/windmill-labs/windmill/commit/7d775b160322e2495d2524a4dde94cc1d2a4718a)) + +## [1.569.0](https://github.com/windmill-labs/windmill/compare/v1.568.0...v1.569.0) (2025-10-30) + + +### Features + +* slack app on workspace level ([#6992](https://github.com/windmill-labs/windmill/issues/6992)) ([700e642](https://github.com/windmill-labs/windmill/commit/700e642c1e0fffae436d574a8c43ba733145f9a7)) +* support slack @ commands ([#7008](https://github.com/windmill-labs/windmill/issues/7008)) ([b0a3da4](https://github.com/windmill-labs/windmill/commit/b0a3da441a899b4990aff1d98008229a7d314cea)) + + +### Bug Fixes + +* **backend:** add locks to inline preprocessor/failure/tool modules in flows ([#6825](https://github.com/windmill-labs/windmill/issues/6825)) ([cf86881](https://github.com/windmill-labs/windmill/commit/cf8688152818d6a35a19ab07f22a413b99ec98df)) +* **frontent:** fix accent destructive color ([#7003](https://github.com/windmill-labs/windmill/issues/7003)) ([3313733](https://github.com/windmill-labs/windmill/commit/33137338f36fe7e424a944651c98f9584e8b4fce)) +* only show if_skipped runs if filter is set to it ([2c4cabb](https://github.com/windmill-labs/windmill/commit/2c4cabb54f38d20f64b19a16a6943e2c29183be4)) +* prioritize dependency jobs by default ([6e87e03](https://github.com/windmill-labs/windmill/commit/6e87e03f364db2a91310735a56ddbcae30eb591e)) +* ui icon nit in instance settings ([#7009](https://github.com/windmill-labs/windmill/issues/7009)) ([dbfaad0](https://github.com/windmill-labs/windmill/commit/dbfaad062390b962e37789bf770e06d33282dca1)) + +## [1.568.0](https://github.com/windmill-labs/windmill/compare/v1.567.3...v1.568.0) (2025-10-30) + + +### Features + +* Add AI_HTTP_HEADERS environment variable for custom AI request headers ([#6994](https://github.com/windmill-labs/windmill/issues/6994)) ([50a476b](https://github.com/windmill-labs/windmill/commit/50a476b529f5afc9a2a4d9721c9c489d3628563b)) +* add cancel_job to windmill python client ([#6995](https://github.com/windmill-labs/windmill/issues/6995)) ([b2a473f](https://github.com/windmill-labs/windmill/commit/b2a473f337826ec1161a6aa224d3dcf1e1b0f0fe)) +* support search for gh repo when pagination needed ([#6982](https://github.com/windmill-labs/windmill/issues/6982)) ([891bf75](https://github.com/windmill-labs/windmill/commit/891bf75519252e408f19a38a7da5fbd784290fca)) + + +### Bug Fixes + +* add missing button on s3FilePicker ([#6993](https://github.com/windmill-labs/windmill/issues/6993)) ([1cddfd1](https://github.com/windmill-labs/windmill/commit/1cddfd1e1aa9215c329b68812eb4c0981b354ce5)) +* auto-unarchived scripts ([#6998](https://github.com/windmill-labs/windmill/issues/6998)) ([83c0c82](https://github.com/windmill-labs/windmill/commit/83c0c8212410e305e5b01ba959dbf1d93bb43572)) +* **backend:** more efficient worker telemetry ([#6997](https://github.com/windmill-labs/windmill/issues/6997)) ([cdf8fdf](https://github.com/windmill-labs/windmill/commit/cdf8fdf4a4249efa22f1e81b3cad38d709656845)) +* improve app evalv2 setting behavior ([abf1b46](https://github.com/windmill-labs/windmill/commit/abf1b46583c7967eef11088acac29c564106cd39)) +* styling on quick search modal input ([#6996](https://github.com/windmill-labs/windmill/issues/6996)) ([8b83ddf](https://github.com/windmill-labs/windmill/commit/8b83ddfb3b9d7e2acd345b98db17c91d14e44e35)) + +## [1.567.3](https://github.com/windmill-labs/windmill/compare/v1.567.2...v1.567.3) (2025-10-28) + + +### Bug Fixes + +* **aiagent:** use tool-based structured output for all claude models ([#6979](https://github.com/windmill-labs/windmill/issues/6979)) ([32180d6](https://github.com/windmill-labs/windmill/commit/32180d636d04896680840cdae15ac8b46f0e53ea)) +* fetch name in saml if present ([91d83d1](https://github.com/windmill-labs/windmill/commit/91d83d1cedd78c5c0c9fcc0cff9cd65a6db22dbc)) +* fix worker tags assignment in edge-cases ([93a5252](https://github.com/windmill-labs/windmill/commit/93a52525adc58b2d9e697780298f91b2b858f940)) + +## [1.567.2](https://github.com/windmill-labs/windmill/compare/v1.567.1...v1.567.2) (2025-10-28) + + +### Bug Fixes + +* add grant all to concurrency_counter ([c78cb1f](https://github.com/windmill-labs/windmill/commit/c78cb1fb7a93a5ecbb4e7e70705fd1147059898a)) + +## [1.567.1](https://github.com/windmill-labs/windmill/compare/v1.567.0...v1.567.1) (2025-10-27) + + +### Bug Fixes + +* avoid listing queued jobs if an end bound is set ([a6a0af3](https://github.com/windmill-labs/windmill/commit/a6a0af3298049dcf93663c5ac659e5e2cabf3ffd)) + +## [1.567.0](https://github.com/windmill-labs/windmill/compare/v1.566.1...v1.567.0) (2025-10-27) + + +### Features + +* is_admin non propagation + change endpoint for forks ([#6958](https://github.com/windmill-labs/windmill/issues/6958)) ([663bc3b](https://github.com/windmill-labs/windmill/commit/663bc3b19ac23938c64b71a0ac10b19d9d6e6d62)) + + +### Bug Fixes + +* **cli:** fix generate-metadata wasm parsers ([a9b99e9](https://github.com/windmill-labs/windmill/commit/a9b99e9467a17f056f3cd325dc5dfee0a5ae9caa)) +* fix windows bun bundler main.ts path detection ([#6962](https://github.com/windmill-labs/windmill/issues/6962)) ([038986d](https://github.com/windmill-labs/windmill/commit/038986d0b593b5aa172f6c1d2d19337ff0239e4c)) +* **frontend:** add transparency to color palette ([#6947](https://github.com/windmill-labs/windmill/issues/6947)) ([a12c278](https://github.com/windmill-labs/windmill/commit/a12c2788ecc1273bb8d98f50520cb1a28e6de280)) +* **frontend:** larger object limit for pdfs and files ([#6961](https://github.com/windmill-labs/windmill/issues/6961)) ([4bf2329](https://github.com/windmill-labs/windmill/commit/4bf23294f8ff4998e75108d974447b3b3e147c45)) +* improve jobs runs page performance ([#6942](https://github.com/windmill-labs/windmill/issues/6942)) ([5b315bf](https://github.com/windmill-labs/windmill/commit/5b315bf2c8e3d992ca690f994cea2205541ab5ae)) + +## [1.566.1](https://github.com/windmill-labs/windmill/compare/v1.566.0...v1.566.1) (2025-10-25) + + +### Bug Fixes + +* fix dependency jobs on CE ([29b6feb](https://github.com/windmill-labs/windmill/commit/29b6febdd81cf16bd14a9ec4528e215e8221704a)) + +## [1.566.0](https://github.com/windmill-labs/windmill/compare/v1.565.0...v1.566.0) (2025-10-24) + + +### Features + +* **aichat:** add user-level custom system prompts ([#6884](https://github.com/windmill-labs/windmill/issues/6884)) ([b5d0f23](https://github.com/windmill-labs/windmill/commit/b5d0f23f4fe9ea05069cada517a47ebe1cf8267a)) +* **backend:** pass all headers and query to webhook preprocessor ([#6931](https://github.com/windmill-labs/windmill/issues/6931)) ([f0bbdd4](https://github.com/windmill-labs/windmill/commit/f0bbdd4aa8a95f1894e9a19687856afa9d73142f)) +* **backend:** stream early return ([#6896](https://github.com/windmill-labs/windmill/issues/6896)) ([e81d629](https://github.com/windmill-labs/windmill/commit/e81d6297050b468e30074ec1ac144be368040d4a)) +* **flow chat:** display image outputs ([#6880](https://github.com/windmill-labs/windmill/issues/6880)) ([53d8fbe](https://github.com/windmill-labs/windmill/commit/53d8fbe5084b3fb8d4eba3555d7d85458bf05aed)) +* job debouncing ([#6878](https://github.com/windmill-labs/windmill/issues/6878)) ([edece03](https://github.com/windmill-labs/windmill/commit/edece035f806aafde5696b0b2711a53ead2bf9c4)) +* support wildcards in http routes ([#6927](https://github.com/windmill-labs/windmill/issues/6927)) ([f43dee1](https://github.com/windmill-labs/windmill/commit/f43dee1952d28a6bbfd2912ab78623da081b60f9)) + + +### Bug Fixes + +* also auto add/del igroup members to workspaces where configured ([#6888](https://github.com/windmill-labs/windmill/issues/6888)) ([1d3245e](https://github.com/windmill-labs/windmill/commit/1d3245eea260609b2f32ba4839e96b2e13d0dabc)) +* apify oauth ([#6902](https://github.com/windmill-labs/windmill/issues/6902)) ([b33e3d8](https://github.com/windmill-labs/windmill/commit/b33e3d85052699becc0d211b2d5737be4d68da4a)) +* **cli:** increase custom bundler output size ([fadfcfd](https://github.com/windmill-labs/windmill/commit/fadfcfd0fce52c3906b1cdd2d2ace5af8dfb01f7)) +* **cli:** resource type sync with protected private hub ([#6933](https://github.com/windmill-labs/windmill/issues/6933)) ([6d403da](https://github.com/windmill-labs/windmill/commit/6d403da05addca02614c9f013a56e78734c0a4f9)) +* **cloud:** better errors when failing to get team plan status ([#6908](https://github.com/windmill-labs/windmill/issues/6908)) ([7b46491](https://github.com/windmill-labs/windmill/commit/7b464915785c576484b53dea8b9a5d9ebd74ec2b)) +* **debouncing:** fix perf issues and re-enable debouncing ([#6932](https://github.com/windmill-labs/windmill/issues/6932)) ([999aaaa](https://github.com/windmill-labs/windmill/commit/999aaaacd8727f2a8afdac2007e6536c66f1c2f7)) +* detect preprocessor in re-exported named exports ([#6899](https://github.com/windmill-labs/windmill/issues/6899)) ([fa76105](https://github.com/windmill-labs/windmill/commit/fa76105d14e6ca113de494d5c3faf8175582712e)) +* **yaml-validator:** update openflow for aiagents ([#6895](https://github.com/windmill-labs/windmill/issues/6895)) ([e40d52d](https://github.com/windmill-labs/windmill/commit/e40d52d411187f9b433c765611686e30a2807037)) + +## [1.565.0](https://github.com/windmill-labs/windmill/compare/v1.564.0...v1.565.0) (2025-10-22) + + +### Features + +* **ai agent:** handle inputTransforms for tools arguments ([#6873](https://github.com/windmill-labs/windmill/issues/6873)) ([2170d8d](https://github.com/windmill-labs/windmill/commit/2170d8dd32f6ab0dd985453a4820c8f228a5929d)) +* UX/UI full overhaul to meet new design system ([032f0c1](https://github.com/windmill-labs/windmill/commit/032f0c1f8c5441955ed6afa0de9be01bbe089c2a)) + + +### Bug Fixes + +* **backend:** batch rerun jobs with preprocessor ([#6875](https://github.com/windmill-labs/windmill/issues/6875)) ([cadf32c](https://github.com/windmill-labs/windmill/commit/cadf32cf039c0da209ca724a609ba0289acc37e0)) +* compute dependencies of apps in deploy to UI ([fc712cf](https://github.com/windmill-labs/windmill/commit/fc712cf2e5a60d2ee2c5ddf8658412e00a99fe13)) +* **parser:** Handle CRLF line endings in bash and PowerShell parsers ([#6889](https://github.com/windmill-labs/windmill/issues/6889)) ([5529736](https://github.com/windmill-labs/windmill/commit/552973678470dc64e7731d2bfc7604cc6660cb03)) +* **windows:** improve 2nd shutdown monitor ([9b1a7f5](https://github.com/windmill-labs/windmill/commit/9b1a7f5a3a8df9b1aeea95b251705156f43fda98)) + +## [1.564.0](https://github.com/windmill-labs/windmill/compare/v1.563.4...v1.564.0) (2025-10-21) + + +### Features + +* add 'on submit' wizard for buttons in app builder ([#6886](https://github.com/windmill-labs/windmill/issues/6886)) ([7fe8494](https://github.com/windmill-labs/windmill/commit/7fe849494d78b043f278b4cb381ac34c88f4566f)) +* **aiagent:** allow mcp as tools ([#6790](https://github.com/windmill-labs/windmill/issues/6790)) ([97ac1be](https://github.com/windmill-labs/windmill/commit/97ac1be036b959e7129a699651c309e283460276)) +* allow optionally forcing ipv4 for reqwest ([#6883](https://github.com/windmill-labs/windmill/issues/6883)) ([d777c77](https://github.com/windmill-labs/windmill/commit/d777c7798b932fc70fbf1477691637a4a0f08177)) +* **flow chat:** add cancel button ([#6869](https://github.com/windmill-labs/windmill/issues/6869)) ([0e98b22](https://github.com/windmill-labs/windmill/commit/0e98b22b429ae968d9a4cd1e544045ead9e9fdfa)) + + +### Bug Fixes + +* debounce_key automatic deletion ([#6885](https://github.com/windmill-labs/windmill/issues/6885)) ([caa21fc](https://github.com/windmill-labs/windmill/commit/caa21fcb33ae4fe2670e54a0f558bacdeff738dd)) +* delete workspace_env on workspace deletion ([7d1ee06](https://github.com/windmill-labs/windmill/commit/7d1ee0662ead211af3c285ccbb2c947314ef67e2)) +* fix column def sync for evalv2 ([1e67073](https://github.com/windmill-labs/windmill/commit/1e670731d5a2e0f94a274c0152cd24518749c306)) +* fix download s3 images with jwt ([df91a79](https://github.com/windmill-labs/windmill/commit/df91a7998c4d802b7f614e026d7c0308a53c8d64)) + +## [1.563.4](https://github.com/windmill-labs/windmill/compare/v1.563.3...v1.563.4) (2025-10-20) + + +### Bug Fixes + +* **cli:** improve generate-flow with raw requirements ([6ccccbc](https://github.com/windmill-labs/windmill/commit/6ccccbcf9e3466597a76175ce02faf7dec8a4ca7)) +* set jwt auth for custom apps ([d86ad75](https://github.com/windmill-labs/windmill/commit/d86ad751d4fb0501964a500dd9db40b9be9c2ae8)) + +## [1.563.3](https://github.com/windmill-labs/windmill/compare/v1.563.2...v1.563.3) (2025-10-18) + + +### Bug Fixes + +* **cli:** when generating flow locks with new inline content, also generate separate content file ([3c114b0](https://github.com/windmill-labs/windmill/commit/3c114b0678531d818e0e46177058425c03a55fc7)) + +## [1.563.2](https://github.com/windmill-labs/windmill/compare/v1.563.1...v1.563.2) (2025-10-17) + + +### Bug Fixes + +* monitor less frequent cleanup process periodicity ([f581ce6](https://github.com/windmill-labs/windmill/commit/f581ce62481a6a62a5f17571f17daf3d99c6c549)) + +## [1.563.1](https://github.com/windmill-labs/windmill/compare/v1.563.0...v1.563.1) (2025-10-17) + + +### Bug Fixes + +* fix concurrency limit behavior with remote agents dep jobs ([6faea9a](https://github.com/windmill-labs/windmill/commit/6faea9adad96ac262477b7e899557ff1ce875b3a)) + +## [1.563.0](https://github.com/windmill-labs/windmill/compare/v1.562.0...v1.563.0) (2025-10-17) + + +### Features + +* **aiagent:** Store AI provider config in localStorage ([#6854](https://github.com/windmill-labs/windmill/issues/6854)) ([93e4b5e](https://github.com/windmill-labs/windmill/commit/93e4b5e0bb8928e912e60028489cf05249c1cb34)) +* http routes streaming ([#6834](https://github.com/windmill-labs/windmill/issues/6834)) ([8cd0006](https://github.com/windmill-labs/windmill/commit/8cd0006498c7afa30a08a0d7cbb2f3a0a9c68994)) + + +### Bug Fixes + +* add grant select on debounce_key ([60e17e5](https://github.com/windmill-labs/windmill/commit/60e17e506806b1fd39cd265952858c068457325c)) +* flow dev mode improvements ([963e0fb](https://github.com/windmill-labs/windmill/commit/963e0fb356c88a29637bd4b8f442540eb718359d)) +* **frontend:** interaction with code instance settings on chrome ([#6859](https://github.com/windmill-labs/windmill/issues/6859)) ([c7fb178](https://github.com/windmill-labs/windmill/commit/c7fb178190a9fa2a4e45322ed016c8c8b7753acf)) + + +### Performance Improvements + +* remove unnecessary db call ([#6853](https://github.com/windmill-labs/windmill/issues/6853)) ([2868eed](https://github.com/windmill-labs/windmill/commit/2868eeda26a35bf8f1b5d39edb4d981ff1181f1f)) + +## [1.562.0](https://github.com/windmill-labs/windmill/compare/v1.561.0...v1.562.0) (2025-10-16) + + +### Features + +* add support for sage intacct oauth ([#6794](https://github.com/windmill-labs/windmill/issues/6794)) ([c86b344](https://github.com/windmill-labs/windmill/commit/c86b3448b86e008f14a25280285cc2f498eb926a)) +* dependency job debouncing ([#6769](https://github.com/windmill-labs/windmill/issues/6769)) ([defb6c9](https://github.com/windmill-labs/windmill/commit/defb6c9694ac294dbf19ba5cd42ce7399ad1b9ac)) + + +### Bug Fixes + +* add configurable timeout sse stream ([f723a1f](https://github.com/windmill-labs/windmill/commit/f723a1fb7227ae45661fea5cf2e6f9928a39672b)) + +## [1.561.0](https://github.com/windmill-labs/windmill/compare/v1.560.0...v1.561.0) (2025-10-16) + + +### Features + +* ansible playbook execution git repo mode (repo viewer + UI utils) ([#6831](https://github.com/windmill-labs/windmill/issues/6831)) ([32fae7a](https://github.com/windmill-labs/windmill/commit/32fae7a10c769473c708970e18c1f8268d62183f)) + + +### Bug Fixes + +* **backend:** revert flow node opti for ai agents ([#6840](https://github.com/windmill-labs/windmill/issues/6840)) ([3b5c962](https://github.com/windmill-labs/windmill/commit/3b5c96247350b70fe947d204e9bff61f81be219c)) +* fix job loader in public apps with jwt token ([a238750](https://github.com/windmill-labs/windmill/commit/a2387505544a04675a7c9fddf2ed8c042f8bfa42)) + +## [1.560.0](https://github.com/windmill-labs/windmill/compare/v1.559.0...v1.560.0) (2025-10-15) + + +### Features + +* add support for zoho oauth ([#6809](https://github.com/windmill-labs/windmill/issues/6809)) ([9d9c29f](https://github.com/windmill-labs/windmill/commit/9d9c29fdfa15cc655854ec909dea944d10ce7374)) +* **backend:** use flow nodes opti for ai agent steps ([#6808](https://github.com/windmill-labs/windmill/issues/6808)) ([8d5acda](https://github.com/windmill-labs/windmill/commit/8d5acda340cd105c5b0dfc2bfe59b7e996bd2707)) +* build pydoc for wmill python client and mount in container image ([#6828](https://github.com/windmill-labs/windmill/issues/6828)) ([d75e9e3](https://github.com/windmill-labs/windmill/commit/d75e9e3d92d43f449a6296b367018f8fa3da6507)) +* **settings:** add unsaved changes warning for workspace settings ([#6813](https://github.com/windmill-labs/windmill/issues/6813)) ([cb88187](https://github.com/windmill-labs/windmill/commit/cb8818796ddd68d2b2ee1dea5f9b0a648f0c1ec9)) + + +### Bug Fixes + +* always create instance groups with uuid ([#6826](https://github.com/windmill-labs/windmill/issues/6826)) ([48acc57](https://github.com/windmill-labs/windmill/commit/48acc57823792c9e795f9735712e1b2ed6d2b4e2)) +* bug for loop flow inconsistent state ([#6815](https://github.com/windmill-labs/windmill/issues/6815)) ([2565222](https://github.com/windmill-labs/windmill/commit/256522273ee65b67075ac91408825b1c6e91ef06)) +* fix concurrency key filter ([892ce64](https://github.com/windmill-labs/windmill/commit/892ce64ea8550c22d65180c71f57c90a65583832)) +* gcp script picker ([#6837](https://github.com/windmill-labs/windmill/issues/6837)) ([d12c8f3](https://github.com/windmill-labs/windmill/commit/d12c8f34efe5ebbdbbf85ae41bb11307dc5d8ea3)) +* resource editor should not autoselect resources for optional fields ([#6821](https://github.com/windmill-labs/windmill/issues/6821)) ([85d1b8a](https://github.com/windmill-labs/windmill/commit/85d1b8a3e6af41bba93128ebcb88ada383ed2d65)) +* support dyn select for sub flow ([#6835](https://github.com/windmill-labs/windmill/issues/6835)) ([b211155](https://github.com/windmill-labs/windmill/commit/b211155784135b1377975a2759f2ddca1cffcea2)) + +## [1.559.0](https://github.com/windmill-labs/windmill/compare/v1.558.1...v1.559.0) (2025-10-14) + + +### Features + +* Add back apply code button in CodeDisplay ([#6800](https://github.com/windmill-labs/windmill/issues/6800)) ([b630208](https://github.com/windmill-labs/windmill/commit/b630208ece8aca9decd30b6964afb88aa8025f41)) +* add support for contextual vars in SQL ([#6791](https://github.com/windmill-labs/windmill/issues/6791)) ([b972eb9](https://github.com/windmill-labs/windmill/commit/b972eb97219d1aba4c820c2e9cd18d519cbd25b4)) +* **rust:** add resource types ([#5843](https://github.com/windmill-labs/windmill/issues/5843)) ([e2feba3](https://github.com/windmill-labs/windmill/commit/e2feba391c47fd62c3ba76556b4c2777e10dc192)) + + +### Bug Fixes + +* **internal:** no max turns ([#6805](https://github.com/windmill-labs/windmill/issues/6805)) ([1a0dbf7](https://github.com/windmill-labs/windmill/commit/1a0dbf7982a647eff40031439e930b1ff53ae615)) +* Safeguard prevents button from deleting non-fork workspaces ([#6795](https://github.com/windmill-labs/windmill/issues/6795)) ([9149faf](https://github.com/windmill-labs/windmill/commit/9149faf3053431633f6046aa3d20ef8fdfb05fea)) +* show workspace prefix to non superadmins for app deploy custom path ([#6793](https://github.com/windmill-labs/windmill/issues/6793)) ([e0d9017](https://github.com/windmill-labs/windmill/commit/e0d90170365d799f05222457feb0fc7f27edf945)) + +## [1.558.1](https://github.com/windmill-labs/windmill/compare/v1.558.0...v1.558.1) (2025-10-09) + + +### Bug Fixes + +* support pg jsonb array ([#6788](https://github.com/windmill-labs/windmill/issues/6788)) ([1c9faf9](https://github.com/windmill-labs/windmill/commit/1c9faf9d03edd663e9a50161b9c37ac24bb3c422)) + +## [1.558.0](https://github.com/windmill-labs/windmill/compare/v1.557.0...v1.558.0) (2025-10-09) + + +### Features + +* allow setting custom cors header on http trigger ([#6786](https://github.com/windmill-labs/windmill/issues/6786)) ([705a177](https://github.com/windmill-labs/windmill/commit/705a1770054ff7f0f65729e9b7c71124bc4bf7a4)) +* **backend:** allow specifying powershell module versions ([#6781](https://github.com/windmill-labs/windmill/issues/6781)) ([bd3e5e6](https://github.com/windmill-labs/windmill/commit/bd3e5e67bbf44f6b1616bca7f926c068427625d4)) +* Database manager for Ducklake instance catalogs ([#6785](https://github.com/windmill-labs/windmill/issues/6785)) ([f798ff4](https://github.com/windmill-labs/windmill/commit/f798ff4535f44ce1c6258ce716277258d2d176ab)) + +## [1.557.0](https://github.com/windmill-labs/windmill/compare/v1.556.1...v1.557.0) (2025-10-09) + + +### Features + +* **flow:** show tool usage in flow conversations ([#6771](https://github.com/windmill-labs/windmill/issues/6771)) ([dc4582a](https://github.com/windmill-labs/windmill/commit/dc4582a1bbc7c0679bfbd4dd139cc14f7cc76ec5)) + + +### Bug Fixes + +* validate that instance group exists before adding to workspace ([#6777](https://github.com/windmill-labs/windmill/issues/6777)) ([b070ed9](https://github.com/windmill-labs/windmill/commit/b070ed955322f01c853427b9f725d104c9c9ea0e)) + +## [1.556.1](https://github.com/windmill-labs/windmill/compare/v1.556.0...v1.556.1) (2025-10-08) + + +### Bug Fixes + +* better handle already completed jobs cases ([8073e5d](https://github.com/windmill-labs/windmill/commit/8073e5daeba451eb130ca3dffd53c6ec3ffb7726)) + +## [1.556.0](https://github.com/windmill-labs/windmill/compare/v1.555.2...v1.556.0) (2025-10-08) + + +### Features + +* add dynamic skip for schedules ([#6739](https://github.com/windmill-labs/windmill/issues/6739)) ([ae8d37f](https://github.com/windmill-labs/windmill/commit/ae8d37fc3478ff689f69b26ee9833281ed5c5311)) +* parallel loop as expr ([#6743](https://github.com/windmill-labs/windmill/issues/6743)) ([27d7809](https://github.com/windmill-labs/windmill/commit/27d7809ef942f75ba2b38ca46209d5e43ba754be)) +* support for protected private hub ([#6762](https://github.com/windmill-labs/windmill/issues/6762)) ([e1d2eb2](https://github.com/windmill-labs/windmill/commit/e1d2eb2870bc358901d6da9b95747e26e1c157c3)) + + +### Bug Fixes + +* better ducklake setup ([#6763](https://github.com/windmill-labs/windmill/issues/6763)) ([258b275](https://github.com/windmill-labs/windmill/commit/258b275f9bfb54323d55001f2adb71a8f55c5a60)) +* correct otel log levels ([#6772](https://github.com/windmill-labs/windmill/issues/6772)) ([c9dae95](https://github.com/windmill-labs/windmill/commit/c9dae9580a195f08245ec047a02dbfc4d3ff823f)) +* fix runnable inputs not being retriggered on change in some rare cases ([50a6106](https://github.com/windmill-labs/windmill/commit/50a61064368ced16e4aa2e24cf80f2478290ca37)) +* fix scheduling of flows with cached results based on inputs ([#6774](https://github.com/windmill-labs/windmill/issues/6774)) ([5ad6182](https://github.com/windmill-labs/windmill/commit/5ad61829e4b60efac2a451d0e8813fda77529efd)) + +## [1.555.2](https://github.com/windmill-labs/windmill/compare/v1.555.1...v1.555.2) (2025-10-06) + + +### Bug Fixes + +* **backend:** use correct ai tool job dir ([#6757](https://github.com/windmill-labs/windmill/issues/6757)) ([7c757b6](https://github.com/windmill-labs/windmill/commit/7c757b68f64bf3cf770cd080c5025f202d40201c)) + +## [1.555.1](https://github.com/windmill-labs/windmill/compare/v1.555.0...v1.555.1) (2025-10-04) + + +### Bug Fixes + +* app button tooltip also when disabled + audit log filters ([#6751](https://github.com/windmill-labs/windmill/issues/6751)) ([605c552](https://github.com/windmill-labs/windmill/commit/605c5526f83d3b985f04570039bb4671cc5912f4)) +* init git repo preview save ([#6753](https://github.com/windmill-labs/windmill/issues/6753)) ([d9c01e0](https://github.com/windmill-labs/windmill/commit/d9c01e0c0392f13e52ad92f87c4216e64e89a3bc)) + +## [1.555.0](https://github.com/windmill-labs/windmill/compare/v1.554.1...v1.555.0) (2025-10-03) + + +### Features + +* end user email env var ([#6750](https://github.com/windmill-labs/windmill/issues/6750)) ([3907c9f](https://github.com/windmill-labs/windmill/commit/3907c9f9512ebd73daf0a2f3ee2e8db6fb9f4df6)) +* **flow:** add option to turn flow into chat ([#6658](https://github.com/windmill-labs/windmill/issues/6658)) ([047420e](https://github.com/windmill-labs/windmill/commit/047420e5ad7b6178291bc7ed75d029794760d18b)) + + +### Bug Fixes + +* **backend:** prevent s3 file upload infinite loop ([#6742](https://github.com/windmill-labs/windmill/issues/6742)) ([6d436d7](https://github.com/windmill-labs/windmill/commit/6d436d745994f954a37f33ca2cc2e9f0801b16b9)) +* show that user is disabled in workspacelist ([#6748](https://github.com/windmill-labs/windmill/issues/6748)) ([c658f32](https://github.com/windmill-labs/windmill/commit/c658f321d68e2d72622d9d167b20cac67364651c)) +* top level assigment doesn't propagate to setContext ([#6745](https://github.com/windmill-labs/windmill/issues/6745)) ([06b152b](https://github.com/windmill-labs/windmill/commit/06b152b295cd4892d7309651d382a05cdcf7d378)) + +## [1.554.1](https://github.com/windmill-labs/windmill/compare/v1.554.0...v1.554.1) (2025-10-02) + + +### Bug Fixes + +* **backend:** concurrency limits preprocessor ([#6727](https://github.com/windmill-labs/windmill/issues/6727)) ([cdb7524](https://github.com/windmill-labs/windmill/commit/cdb75241188ee0a6d7bc62ca6dd639606eef426a)) + +## [1.554.0](https://github.com/windmill-labs/windmill/compare/v1.553.0...v1.554.0) (2025-10-01) + + +### Features + +* **cli:** allow skipping branch validation ([#6721](https://github.com/windmill-labs/windmill/issues/6721)) ([9e6ceba](https://github.com/windmill-labs/windmill/commit/9e6cebac557fd9a8530df54c227e915435fd2de5)) + + +### Bug Fixes + +* allow running scripts in json view ([74a7543](https://github.com/windmill-labs/windmill/commit/74a75431c072e9f084a43b8d8f195ca7757faf07)) +* fix job duration unwrap crash ([e2e3ae9](https://github.com/windmill-labs/windmill/commit/e2e3ae9f0280d4369f77064ed85e5fde7e9d5a0d)) + +## [1.553.0](https://github.com/windmill-labs/windmill/compare/v1.552.1...v1.553.0) (2025-09-30) + + +### Features + +* **backend:** allow multiple workspaces in jwt ([#6714](https://github.com/windmill-labs/windmill/issues/6714)) ([526dfd7](https://github.com/windmill-labs/windmill/commit/526dfd72377b90fb47b9c2c3924a1377e2715ae1)) +* **backend:** array and object params support in pwsh ([#6706](https://github.com/windmill-labs/windmill/issues/6706)) ([898eb62](https://github.com/windmill-labs/windmill/commit/898eb6231beb5ca45d59da069f458f7828f427f0)) +* support esm mode for codebase bundles ([#6709](https://github.com/windmill-labs/windmill/issues/6709)) ([d382ea7](https://github.com/windmill-labs/windmill/commit/d382ea7c8b372471dd3393720ff93749fde898f5)) + + +### Bug Fixes + +* multiselect + jsoneditor nits ([5aeb3fa](https://github.com/windmill-labs/windmill/commit/5aeb3fa0b74fb0d72f0439ce540baf224654f1ae)) + +## [1.552.1](https://github.com/windmill-labs/windmill/compare/v1.552.0...v1.552.1) (2025-09-29) + + +### Bug Fixes + +* fix c# with nsjail ([2055e53](https://github.com/windmill-labs/windmill/commit/2055e536a7fcb9cfe155c0fa67de6ae49d925f97)) +* **frontend:** allow dates before 2000 in date picker ([#6707](https://github.com/windmill-labs/windmill/issues/6707)) ([ce653f8](https://github.com/windmill-labs/windmill/commit/ce653f8a0538fcc88ef78f6c50960a7340648b0f)) + +## [1.552.0](https://github.com/windmill-labs/windmill/compare/v1.551.4...v1.552.0) (2025-09-29) + + +### Features + +* powershell private repo support ([#6684](https://github.com/windmill-labs/windmill/issues/6684)) ([4bbbeb9](https://github.com/windmill-labs/windmill/commit/4bbbeb956f8f09ea5a8af241912a1bead1e06520)) + + +### Bug Fixes + +* external links in critical alert ([e2608f9](https://github.com/windmill-labs/windmill/commit/e2608f9aacd30e2a7aeb5b850802514f64a41380)) +* fix app schema form rendering ([481c877](https://github.com/windmill-labs/windmill/commit/481c8775377f7f01ad01b5db85a98ccffadada91)) +* **frontend:** prevent label interference with monaco editor in instance settings ([#6701](https://github.com/windmill-labs/windmill/issues/6701)) ([c751a5d](https://github.com/windmill-labs/windmill/commit/c751a5d6aa49e4bc0970f87b3f1e975e8ee58479)) +* **mcp:** filter out tools with long names ([#6692](https://github.com/windmill-labs/windmill/issues/6692)) ([cc2afdb](https://github.com/windmill-labs/windmill/commit/cc2afdb264b0eaa353e5f2736c98e475337b71f7)) +* show more autoscaling events ([#6704](https://github.com/windmill-labs/windmill/issues/6704)) ([d56dea4](https://github.com/windmill-labs/windmill/commit/d56dea4969ed5c6eec30c72cf9f1171889444007)) +* **uv:** log stdout on `uv pip install` error ([#6702](https://github.com/windmill-labs/windmill/issues/6702)) ([5f63ce6](https://github.com/windmill-labs/windmill/commit/5f63ce6dd8697533de1e0af786e463f3224912c2)) + +## [1.551.4](https://github.com/windmill-labs/windmill/compare/v1.551.3...v1.551.4) (2025-09-29) + + +### Bug Fixes + +* migrate dotnet from msft images to script install ([cfec8e9](https://github.com/windmill-labs/windmill/commit/cfec8e99fb55928dfed3b7e80fb63cc279553dec)) + +## [1.551.3](https://github.com/windmill-labs/windmill/compare/v1.551.2...v1.551.3) (2025-09-29) + + +### Bug Fixes + +* migrate dotnet from bitnami to microsoft images ([5ae525a](https://github.com/windmill-labs/windmill/commit/5ae525a9f14de20d45e6075baa979eb4aaac4850)) + +## [1.551.2](https://github.com/windmill-labs/windmill/compare/v1.551.1...v1.551.2) (2025-09-29) + + +### Bug Fixes + +* fix copy first step input ([81616cb](https://github.com/windmill-labs/windmill/commit/81616cbe1e27bc3f45cfa35cd359ce9a0f493f35)) + +## [1.551.1](https://github.com/windmill-labs/windmill/compare/v1.551.0...v1.551.1) (2025-09-28) + + +### Bug Fixes + +* buttons are back to semi-bold ([bdd36c0](https://github.com/windmill-labs/windmill/commit/bdd36c0b4d5c590e66bf471c8e4b5f681b9464aa)) + +## [1.551.0](https://github.com/windmill-labs/windmill/compare/v1.550.0...v1.551.0) (2025-09-27) + + +### Features + +* UX improvements (all inputs) ([72b744c](https://github.com/windmill-labs/windmill/commit/72b744c4e1bc3c1f3098f4d6de6c9474d7a8fb84)) + +## [1.550.0](https://github.com/windmill-labs/windmill/compare/v1.549.1...v1.550.0) (2025-09-27) + + +### Features + +* ai agent streaming ([#6644](https://github.com/windmill-labs/windmill/issues/6644)) ([f990107](https://github.com/windmill-labs/windmill/commit/f990107c45fbb2e955ef67439e92328976091eb0)) + + +### Bug Fixes + +* improve behavior for already completed jobs when doing immediate cancels ([341cdcf](https://github.com/windmill-labs/windmill/commit/341cdcf66efdfd504492be32d9b4f5cb9db2df2a)) +* improve dyn select as flow input ([6ece0ac](https://github.com/windmill-labs/windmill/commit/6ece0ac5758d4f5e8c0d55ee78a524e454ad264b)) +* improve graph rendering performances ([7add574](https://github.com/windmill-labs/windmill/commit/7add57499c02ac53a7f7adbabbb279d7c41ab275)) +* improve performance of flow viewer ([311b410](https://github.com/windmill-labs/windmill/commit/311b410f2f65c3bdfc483c80cc5ef72b6864118a)) +* limit auto data tables to tables of col < 100 ([f28ed9a](https://github.com/windmill-labs/windmill/commit/f28ed9a5f5c6032c49734e6770c2f2c9e2e4a001)) +* make schedule more resilient in case of pg clock shifts ([8786130](https://github.com/windmill-labs/windmill/commit/87861301f28cab136fe7af094690539e3daa613f)) +* restore set_progress feature with sse ([7df13b3](https://github.com/windmill-labs/windmill/commit/7df13b3e7bb095475d0fe54b7f635e3861fb0f73)) +* scim group/users audit logs ([#6682](https://github.com/windmill-labs/windmill/issues/6682)) ([ca4f9ee](https://github.com/windmill-labs/windmill/commit/ca4f9ee8c12f01fc7c3bcedf5d41e59dc28eb1f2)) +* support label + value for dynamic enums of selects ([ec9e5a9](https://github.com/windmill-labs/windmill/commit/ec9e5a9acbd352b20399d403be55361c73084aff)) + +## [1.549.1](https://github.com/windmill-labs/windmill/compare/v1.549.0...v1.549.1) (2025-09-26) + + +### Bug Fixes + +* do not request unecessarily get_scheduled_for ([0269211](https://github.com/windmill-labs/windmill/commit/02692111a1a8eefb2675b14d53f109a66c1b9a78)) +* fix agent_workers completed job back-compatibility deserialization ([db4bc7e](https://github.com/windmill-labs/windmill/commit/db4bc7ee6963955abc7e290bd67ea913b0f5e2ad)) + +## [1.549.0](https://github.com/windmill-labs/windmill/compare/v1.548.3...v1.549.0) (2025-09-26) + + +### Features + +* **backend:** job result stream optimization ([#6673](https://github.com/windmill-labs/windmill/issues/6673)) ([8f4fef9](https://github.com/windmill-labs/windmill/commit/8f4fef98042c49346c89bdf5e0b9b1f2d52e371f)) + + +### Bug Fixes + +* scim group handling when deleting instance user + conversion ([#6677](https://github.com/windmill-labs/windmill/issues/6677)) ([4205e83](https://github.com/windmill-labs/windmill/commit/4205e83cfde453827eab23c31e76a0f0490d31b7)) + +## [1.548.3](https://github.com/windmill-labs/windmill/compare/v1.548.2...v1.548.3) (2025-09-25) + + +### Bug Fixes + +* fix job loader token initialization ([f5d238e](https://github.com/windmill-labs/windmill/commit/f5d238edcfed6b0f066d459cdc718679a7b51187)) +* websocket runnable [#6675](https://github.com/windmill-labs/windmill/issues/6675) ([a308782](https://github.com/windmill-labs/windmill/commit/a308782bcf7ef9913887521d74796b490619d0c8)) + +## [1.548.2](https://github.com/windmill-labs/windmill/compare/v1.548.1...v1.548.2) (2025-09-24) + + +### Bug Fixes + +* **ui:** workers button on navbar require a single click only ([afa8104](https://github.com/windmill-labs/windmill/commit/afa8104cb0c1a8f1a6fe124a6e01c1d32f049afa)) + +## [1.548.1](https://github.com/windmill-labs/windmill/compare/v1.548.0...v1.548.1) (2025-09-24) + + +### Bug Fixes + +* improve vscode dev mode for flows ([eda985d](https://github.com/windmill-labs/windmill/commit/eda985df1cce70ea3ce4117577c889a3dbc47c6a)) +* improve vscode dev mode layout for scripts ([574364a](https://github.com/windmill-labs/windmill/commit/574364af050f2cc66c986fed8001409aa48f3530)) + +## [1.548.0](https://github.com/windmill-labs/windmill/compare/v1.547.0...v1.548.0) (2025-09-24) + + +### Features + +* app button run in background option ([#6670](https://github.com/windmill-labs/windmill/issues/6670)) ([6b61262](https://github.com/windmill-labs/windmill/commit/6b61262603b247da717d9fd188746078ea779c34)) +* websocket trigger send runnable result even if error ([#6664](https://github.com/windmill-labs/windmill/issues/6664)) ([ef75ed3](https://github.com/windmill-labs/windmill/commit/ef75ed3df7bf99e735a579291078f4ea9db4fcf6)) + + +### Bug Fixes + +* **aichat:** in script mode use diff based edits for good providers only ([#6665](https://github.com/windmill-labs/windmill/issues/6665)) ([f66f131](https://github.com/windmill-labs/windmill/commit/f66f131fed88a71f00d2cadb404ed4fa7698deb6)) +* **backend:** rework `dependency_map` handling ([#6598](https://github.com/windmill-labs/windmill/issues/6598)) ([ed806bf](https://github.com/windmill-labs/windmill/commit/ed806bf9d07de9f22c8a00260984e94eafd6ddf8)) +* fix vscode extension dev mode ([31c2e36](https://github.com/windmill-labs/windmill/commit/31c2e3662f53e6acb6b290f8128cec4a9a98bf73)) +* flow quick picker refresh ([#6666](https://github.com/windmill-labs/windmill/issues/6666)) ([b0e7577](https://github.com/windmill-labs/windmill/commit/b0e7577955c954fef68d0a9d7218f5891100e1ab)) + +## [1.547.0](https://github.com/windmill-labs/windmill/compare/v1.546.1...v1.547.0) (2025-09-23) + + +### Features + +* add dyn select for flow step [#6662](https://github.com/windmill-labs/windmill/issues/6662) ([b64e509](https://github.com/windmill-labs/windmill/commit/b64e509e60fadc631ccd6090654d523d08c06e35)) + + +### Bug Fixes + +* **cli:** improve result printing of the CLI ([a7cbc28](https://github.com/windmill-labs/windmill/commit/a7cbc289af1eaacbb50d53f2bcb4a14f50d420ef)) +* improve scripts duplicity error in global search ([2de7134](https://github.com/windmill-labs/windmill/commit/2de7134b85d8249f661e3db34ff43c29b13fa0aa)) + +## [1.546.1](https://github.com/windmill-labs/windmill/compare/v1.546.0...v1.546.1) (2025-09-23) + + +### Bug Fixes + +* **mcp:** use stateless mode for openai sdk compatibility ([#6656](https://github.com/windmill-labs/windmill/issues/6656)) ([389b692](https://github.com/windmill-labs/windmill/commit/389b692523507a28916e96b481c60f3c49cd31da)) + +## [1.546.0](https://github.com/windmill-labs/windmill/compare/v1.545.0...v1.546.0) (2025-09-23) + + +### Features + +* app builder button tooltip ([#6652](https://github.com/windmill-labs/windmill/issues/6652)) ([08952c6](https://github.com/windmill-labs/windmill/commit/08952c6c6e0afdde8fc941f9f1d17870fe25878a)) +* dynamically hide tabs in app builder ([#6653](https://github.com/windmill-labs/windmill/issues/6653)) ([de7251d](https://github.com/windmill-labs/windmill/commit/de7251d85734757a1f3e222c715f807ba167d535)) +* split RUST_LOG into RUST_LOG and RUST_LOG_STDOUT ([7a13e9e](https://github.com/windmill-labs/windmill/commit/7a13e9e98840a456ef6625cea838e3e82def5c4b)) + + +### Bug Fixes + +* add settable poll delay for sse streams ([0392103](https://github.com/windmill-labs/windmill/commit/039210369383bcc3a15d95cba9efb591ee8e9891)) +* cli path on windows + error_handler_muted_on_cancel ([#6657](https://github.com/windmill-labs/windmill/issues/6657)) ([6ba3a43](https://github.com/windmill-labs/windmill/commit/6ba3a4397e439d40079d524de15507257442c5e1)) +* improve reliability of exits in case graceful handler didn't exit as expected ([f6dd78c](https://github.com/windmill-labs/windmill/commit/f6dd78cb11ee73408f66b4670c395ade99beedbe)) + +## [1.545.0](https://github.com/windmill-labs/windmill/compare/v1.544.2...v1.545.0) (2025-09-20) + + +### Features + +* force cancel in batch cancel ([2eeaf56](https://github.com/windmill-labs/windmill/commit/2eeaf5639dfcdb0114dc5fc323dbd10ef5ceaca5)) +* load for loop jobs timeline directly from for loop flow status ([#6646](https://github.com/windmill-labs/windmill/issues/6646)) ([f71f9b0](https://github.com/windmill-labs/windmill/commit/f71f9b089438fc29dfb26a21ec2b8aa5867d0296)) + + +### Bug Fixes + +* add termination handler earlier in lifecycle ([56cdb69](https://github.com/windmill-labs/windmill/commit/56cdb69e599e295a51284c58d6ff56498ad22fd7)) +* allow variable picker in password field ([194887e](https://github.com/windmill-labs/windmill/commit/194887e97b3250ef28b0ddf5c03932148858ca24)) +* fix flow quick picker stuck ([#6638](https://github.com/windmill-labs/windmill/issues/6638)) ([5cab802](https://github.com/windmill-labs/windmill/commit/5cab802c421c5aeda8c2b57242a4e0c4c074a5f8)) +* fix too strict aggrid coldef validation ([612d003](https://github.com/windmill-labs/windmill/commit/612d00367c2a288920c14449561433d199b95cae)) +* retry python relative imports on errno 104 ([81ff0dc](https://github.com/windmill-labs/windmill/commit/81ff0dcd8cc2fb1c5fc12a558702cbc950ca1eb4)) +* teams api improvements ([#6643](https://github.com/windmill-labs/windmill/issues/6643)) ([70e9ae1](https://github.com/windmill-labs/windmill/commit/70e9ae14a9541862aaa31b8435abb0185805d4b7)) + +## [1.544.2](https://github.com/windmill-labs/windmill/compare/v1.544.1...v1.544.2) (2025-09-18) + + +### Bug Fixes + +* improve flowtimeline ([74a8d8a](https://github.com/windmill-labs/windmill/commit/74a8d8a6f7cc7e16f23d6f7999e51e9f2f6581d4)) + +## [1.544.1](https://github.com/windmill-labs/windmill/compare/v1.544.0...v1.544.1) (2025-09-18) + + +### Bug Fixes + +* fix onLoad auth issue ([bb4699b](https://github.com/windmill-labs/windmill/commit/bb4699bdc6d4e458175594db92b34fdf12fadf1d)) + +## [1.544.0](https://github.com/windmill-labs/windmill/compare/v1.543.0...v1.544.0) (2025-09-18) + + +### Features + +* **ai agent:** allow multiple images input for ai agent + code cleaning ([#6591](https://github.com/windmill-labs/windmill/issues/6591)) ([3199f9f](https://github.com/windmill-labs/windmill/commit/3199f9fffd36c84d7cbb4a512935f7eb19fa5049)) +* **aichat:** add max tokens settings ([#6613](https://github.com/windmill-labs/windmill/issues/6613)) ([d837bad](https://github.com/windmill-labs/windmill/commit/d837badf2c70c483e260b099a663fdda3f1f509a)) +* allow operator to use script/flow with dynselect input ([#6616](https://github.com/windmill-labs/windmill/issues/6616)) ([e98bde6](https://github.com/windmill-labs/windmill/commit/e98bde6be6633e9a529b67672fcde5e4ce5f9129)) +* **backend:** flow streaming ([#6520](https://github.com/windmill-labs/windmill/issues/6520)) ([993baf4](https://github.com/windmill-labs/windmill/commit/993baf46bd46524b5cefe1d6ffc037e9c6ec32d2)) +* fix gcp cleanup and add ack-deadline for gcp push delivery [#6631](https://github.com/windmill-labs/windmill/issues/6631) ([4b71495](https://github.com/windmill-labs/windmill/commit/4b7149527b52ac42094af580e3788326c30a0c70)) +* **flow:** Add helper to add expression to arrays ([#6629](https://github.com/windmill-labs/windmill/issues/6629)) ([56ddad2](https://github.com/windmill-labs/windmill/commit/56ddad2d5a960c782fad89c2c86ad9cc2c4dd8cb)) +* **frontend:** allow publishing script to hub from list view ([#6634](https://github.com/windmill-labs/windmill/issues/6634)) ([39b2f54](https://github.com/windmill-labs/windmill/commit/39b2f547799a6d75012ca2595d532e87a3035e85)) +* simplify sync vs promotion mode ui in git sync settings ([#6615](https://github.com/windmill-labs/windmill/issues/6615)) ([7707bb8](https://github.com/windmill-labs/windmill/commit/7707bb8fecd85cc65a4e511f3033170a1c24fb1d)) +* update git sync script for email triggers ([#6582](https://github.com/windmill-labs/windmill/issues/6582)) ([e97c535](https://github.com/windmill-labs/windmill/commit/e97c535376177b8681fbb6b6553d6f241462f7a9)) + + +### Bug Fixes + +* add ack deadline gcp ([#6625](https://github.com/windmill-labs/windmill/issues/6625)) ([426065e](https://github.com/windmill-labs/windmill/commit/426065efee5e5e775dde403f7e7f7c78d002909a)) +* **aiagent:** fix endpoint for azure ([#6633](https://github.com/windmill-labs/windmill/issues/6633)) ([709a937](https://github.com/windmill-labs/windmill/commit/709a937ac2dab3d8a18b01f200976fa2d4625e89)) +* Don't reencrypt secrets on workspace forking ([#6622](https://github.com/windmill-labs/windmill/issues/6622)) ([9325f56](https://github.com/windmill-labs/windmill/commit/9325f5636c2e957a724bb051d41e15d2966b2899)) +* jumpcloud scim support + instance settings ui bug (nextcloud oauth) ([#6618](https://github.com/windmill-labs/windmill/issues/6618)) ([9ff4ca0](https://github.com/windmill-labs/windmill/commit/9ff4ca06629a0cf7da2996f2b22ea7915cc4705e)) + +## [1.543.0](https://github.com/windmill-labs/windmill/compare/v1.542.4...v1.543.0) (2025-09-15) + + +### Features + +* **ai agent:** handle images in ai agent ([#6572](https://github.com/windmill-labs/windmill/issues/6572)) ([20f48e6](https://github.com/windmill-labs/windmill/commit/20f48e6dedde8b494367f962e870a0ef7ab85e67)) + + +### Bug Fixes + +* fix navbapp app navigation ([223feed](https://github.com/windmill-labs/windmill/commit/223feede4d2cf5cb7a3079995ca5cb0a01a87f35)) +* **frontend:** add timeline to the flow log viewer ([#6577](https://github.com/windmill-labs/windmill/issues/6577)) ([b0495b7](https://github.com/windmill-labs/windmill/commit/b0495b7133550ecbfd04c8cc90dcf9e9ca57f99e)) +* run preprocessor even if empty flow ([#6609](https://github.com/windmill-labs/windmill/issues/6609)) ([c24c629](https://github.com/windmill-labs/windmill/commit/c24c6293179e584fbeddbe787176db6fc748cf4a)) + +## [1.542.4](https://github.com/windmill-labs/windmill/compare/v1.542.3...v1.542.4) (2025-09-13) + + +### Bug Fixes + +* **aichat:** fix tool usage for gemini models ([#6599](https://github.com/windmill-labs/windmill/issues/6599)) ([7dbf5ca](https://github.com/windmill-labs/windmill/commit/7dbf5ca561a4045a829a8642491ed242243ec825)) +* allow custom models in ai agent step ([eb7cbd2](https://github.com/windmill-labs/windmill/commit/eb7cbd29bf48a5587791ef586f642af56a4ab11c)) +* **backend:** email trigger fix build ([#6602](https://github.com/windmill-labs/windmill/issues/6602)) ([82dcb71](https://github.com/windmill-labs/windmill/commit/82dcb711ca46635683f51dfab9a67204662ddab9)) +* **backend:** email triggers error handler and retry ([#6601](https://github.com/windmill-labs/windmill/issues/6601)) ([41667d0](https://github.com/windmill-labs/windmill/commit/41667d06fc8552dbc0235feb7f297f18e060b01d)) +* custom tag helper ([bef6bb8](https://github.com/windmill-labs/windmill/commit/bef6bb826f1c72d91130fc6886bf062ebf809c0c)) +* fix first step's schema clone ([84757a6](https://github.com/windmill-labs/windmill/commit/84757a68d7122f6b1489cdaec417e9adcd827555)) +* improve aggrid actions column ([#6600](https://github.com/windmill-labs/windmill/issues/6600)) ([c755e2b](https://github.com/windmill-labs/windmill/commit/c755e2bad006a5cc147adb47417270af9d042d24)) +* use $var: syntax for empty string template fields ([#6603](https://github.com/windmill-labs/windmill/issues/6603)) ([0a7d762](https://github.com/windmill-labs/windmill/commit/0a7d762010002da129856414b4d3300bce09ac28)) +* workspace forks script versioning (hashes) ([#6604](https://github.com/windmill-labs/windmill/issues/6604)) ([9454ab5](https://github.com/windmill-labs/windmill/commit/9454ab5cc439aeba7eb9133bd34cfa5a9e27fcaf)) + +## [1.542.3](https://github.com/windmill-labs/windmill/compare/v1.542.2...v1.542.3) (2025-09-11) + + +### Bug Fixes + +* catchPanicLayer to handle axum panics more gracefully + onFailure tracing ([a8f67f4](https://github.com/windmill-labs/windmill/commit/a8f67f483c4a14cc80e529ae0add0e6678d4dd30)) +* **perf:** improve perf and reliablity using tcp_nodelay and content-length for intra worker requests ([6c34cd8](https://github.com/windmill-labs/windmill/commit/6c34cd8ad672058d7793b8727247d80e6afe7531)) +* scim members optional (jumpcloud) ([#6579](https://github.com/windmill-labs/windmill/issues/6579)) ([cb54437](https://github.com/windmill-labs/windmill/commit/cb54437e739570c7de6428348a5a3859543c5f3b)) + +## [1.542.2](https://github.com/windmill-labs/windmill/compare/v1.542.1...v1.542.2) (2025-09-11) + + +### Bug Fixes + +* archive by hash workspace specificity ([0518c46](https://github.com/windmill-labs/windmill/commit/0518c46059e934fe14e1dddac7d0bab3c5907c90)) + ## [1.542.1](https://github.com/windmill-labs/windmill/compare/v1.542.0...v1.542.1) (2025-09-11) diff --git a/Dockerfile b/Dockerfile index 8588af2569..b3f18b4e17 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG DEBIAN_IMAGE=debian:bookworm-slim -ARG RUST_IMAGE=rust:1.88-slim-bookworm +ARG RUST_IMAGE=rust:1.90-slim-bookworm # Build libwindmill_duckdb_ffi_internal.so separately FROM ${RUST_IMAGE} AS windmill_duckdb_ffi_internal_builder @@ -30,7 +30,7 @@ WORKDIR /windmill ENV SQLX_OFFLINE=true # ENV CARGO_INCREMENTAL=1 -FROM node:20-alpine as frontend +FROM node:24-alpine as frontend # install dependencies WORKDIR /frontend @@ -48,6 +48,7 @@ COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi. RUN cd /backend/windmill-api && . ./build_openapi.sh COPY /backend/parsers/windmill-parser-wasm/pkg/ /backend/parsers/windmill-parser-wasm/pkg/ COPY /typescript-client/docs/ /frontend/static/tsdocs/ +COPY /python-client/docs/ /frontend/static/pydocs/ RUN npm run generate-backend-client ENV NODE_OPTIONS "--max-old-space-size=8192" @@ -204,7 +205,7 @@ COPY --from=windmill_duckdb_ffi_internal_builder /windmill-duckdb-ffi-internal/t COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno -COPY --from=oven/bun:1.2.18 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.2.23 /usr/local/bin/bun /usr/bin/bun COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer diff --git a/backend/.gitignore b/backend/.gitignore index f96e7e106d..6645e22618 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -9,4 +9,5 @@ windmill-api/openapi-*.* .duckdb/* *ee.rs generate_mcp_endpoints_tools/venv -bacon.toml \ No newline at end of file +bacon.toml +libwindmill_duckdb_ffi_internal.so \ No newline at end of file diff --git a/backend/.sqlx/query-0084c1246d1391d106da2e67a394eafc6695257632406ed9a2111dba1dd106c7.json b/backend/.sqlx/query-0084c1246d1391d106da2e67a394eafc6695257632406ed9a2111dba1dd106c7.json new file mode 100644 index 0000000000..f91485e855 --- /dev/null +++ b/backend/.sqlx/query-0084c1246d1391d106da2e67a394eafc6695257632406ed9a2111dba1dd106c7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args as \"args: sqlx::types::Json>>\" FROM v2_job WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args: sqlx::types::Json>>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "0084c1246d1391d106da2e67a394eafc6695257632406ed9a2111dba1dd106c7" +} diff --git a/backend/.sqlx/query-00b9f392a5cc07bd4ed14e3b69f96408e219d70015dd2f419fc87a440f070c64.json b/backend/.sqlx/query-00b9f392a5cc07bd4ed14e3b69f96408e219d70015dd2f419fc87a440f070c64.json new file mode 100644 index 0000000000..f09f6d0d85 --- /dev/null +++ b/backend/.sqlx/query-00b9f392a5cc07bd4ed14e3b69f96408e219d70015dd2f419fc87a440f070c64.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id)\n SELECT $1, importer_path, importer_kind, imported_path, importer_node_id\n FROM dependency_map\n WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "00b9f392a5cc07bd4ed14e3b69f96408e219d70015dd2f419fc87a440f070c64" +} diff --git a/backend/.sqlx/query-01050e7057f3d1971ad9e47ac83bf6a3c3c9f41689c3607f0b264437ae6b3324.json b/backend/.sqlx/query-01050e7057f3d1971ad9e47ac83bf6a3c3c9f41689c3607f0b264437ae6b3324.json new file mode 100644 index 0000000000..1af0e078d6 --- /dev/null +++ b/backend/.sqlx/query-01050e7057f3d1971ad9e47ac83bf6a3c3c9f41689c3607f0b264437ae6b3324.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT data FROM app_bundles WHERE app_version_id = $1 AND file_type = $2 AND w_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "data", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "01050e7057f3d1971ad9e47ac83bf6a3c3c9f41689c3607f0b264437ae6b3324" +} diff --git a/backend/.sqlx/query-011c7638eeeda710deb86a216a9e10df9c3e9458e85bcdde466b01011a1f2ac2.json b/backend/.sqlx/query-011c7638eeeda710deb86a216a9e10df9c3e9458e85bcdde466b01011a1f2ac2.json deleted file mode 100644 index 749b684fd9..0000000000 --- a/backend/.sqlx/query-011c7638eeeda710deb86a216a9e10df9c3e9458e85bcdde466b01011a1f2ac2.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n path,\n is_flow,\n workspace_id,\n owner,\n email,\n trigger_config as \"trigger_config!: _\"\n FROM\n capture_config\n WHERE\n trigger_kind = 'postgres' AND\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n trigger_config IS NOT NULL AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "owner", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "trigger_config!: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - false, - true - ] - }, - "hash": "011c7638eeeda710deb86a216a9e10df9c3e9458e85bcdde466b01011a1f2ac2" -} diff --git a/backend/.sqlx/query-0186c1058f147e012b8120c342caf8688a6d1643747be3ec4f784c3029a59e52.json b/backend/.sqlx/query-0186c1058f147e012b8120c342caf8688a6d1643747be3ec4f784c3029a59e52.json new file mode 100644 index 0000000000..1880104006 --- /dev/null +++ b/backend/.sqlx/query-0186c1058f147e012b8120c342caf8688a6d1643747be3ec4f784c3029a59e52.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT j.id\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE r.ping < now() - ($1 || ' seconds')::interval\n AND q.running = true AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') AND j.same_worker = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0186c1058f147e012b8120c342caf8688a6d1643747be3ec4f784c3029a59e52" +} diff --git a/backend/.sqlx/query-031d0d70b0aff52feaad487bddb74e5ef0aaa2505facbea8c764003dfc8fffb1.json b/backend/.sqlx/query-031d0d70b0aff52feaad487bddb74e5ef0aaa2505facbea8c764003dfc8fffb1.json deleted file mode 100644 index 8fce6ae729..0000000000 --- a/backend/.sqlx/query-031d0d70b0aff52feaad487bddb74e5ef0aaa2505facbea8c764003dfc8fffb1.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket' AND server_id = $5 AND last_client_ping > NOW() - INTERVAL '10 seconds' RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "031d0d70b0aff52feaad487bddb74e5ef0aaa2505facbea8c764003dfc8fffb1" -} diff --git a/backend/.sqlx/query-045b26db0cefe6eaac0e572661d984ff5ce7086ac511e8647e2024d9dbe0af56.json b/backend/.sqlx/query-045b26db0cefe6eaac0e572661d984ff5ce7086ac511e8647e2024d9dbe0af56.json new file mode 100644 index 0000000000..9affa78033 --- /dev/null +++ b/backend/.sqlx/query-045b26db0cefe6eaac0e572661d984ff5ce7086ac511e8647e2024d9dbe0af56.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "045b26db0cefe6eaac0e572661d984ff5ce7086ac511e8647e2024d9dbe0af56" +} diff --git a/backend/.sqlx/query-db558b5ecdc4c3b1af0def511f1bcd91a548f00376f644c8ba38f73812b462d0.json b/backend/.sqlx/query-07335b75233811352fb898cf3d6c8fe7fd014adbf40cc4bc8c041f5864423367.json similarity index 62% rename from backend/.sqlx/query-db558b5ecdc4c3b1af0def511f1bcd91a548f00376f644c8ba38f73812b462d0.json rename to backend/.sqlx/query-07335b75233811352fb898cf3d6c8fe7fd014adbf40cc4bc8c041f5864423367.json index 1e36101464..9c85043e7c 100644 --- a/backend/.sqlx/query-db558b5ecdc4c3b1af0def511f1bcd91a548f00376f644c8ba38f73812b462d0.json +++ b/backend/.sqlx/query-07335b75233811352fb898cf3d6c8fe7fd014adbf40cc4bc8c041f5864423367.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL DO UPDATE SET deployment_msg = $4", + "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL\n DO UPDATE SET deployment_msg = EXCLUDED.deployment_msg", "describe": { "columns": [], "parameters": { @@ -13,5 +13,5 @@ }, "nullable": [] }, - "hash": "db558b5ecdc4c3b1af0def511f1bcd91a548f00376f644c8ba38f73812b462d0" + "hash": "07335b75233811352fb898cf3d6c8fe7fd014adbf40cc4bc8c041f5864423367" } diff --git a/backend/.sqlx/query-081f838b3dbe81631d17e7ca0751db725a7f92d4e43a86bcfa06a4ac7c70ac8f.json b/backend/.sqlx/query-081f838b3dbe81631d17e7ca0751db725a7f92d4e43a86bcfa06a4ac7c70ac8f.json new file mode 100644 index 0000000000..75f021b168 --- /dev/null +++ b/backend/.sqlx/query-081f838b3dbe81631d17e7ca0751db725a7f92d4e43a86bcfa06a4ac7c70ac8f.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET flow_status = \n CASE WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN\n JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status,\n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT],\n $4\n ),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $5),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $6)\n ELSE\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4)\n END\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "081f838b3dbe81631d17e7ca0751db725a7f92d4e43a86bcfa06a4ac7c70ac8f" +} diff --git a/backend/.sqlx/query-a59b70164dc87224d09a04d5469ca217eb19a15a250c3b83ca63f606f89b9681.json b/backend/.sqlx/query-08c1121171b98889f188ea6b33b1861f3483fa70b5d58dd2838a5cb6dabe9cc1.json similarity index 59% rename from backend/.sqlx/query-a59b70164dc87224d09a04d5469ca217eb19a15a250c3b83ca63f606f89b9681.json rename to backend/.sqlx/query-08c1121171b98889f188ea6b33b1861f3483fa70b5d58dd2838a5cb6dabe9cc1.json index da8842f9a7..16a6101127 100644 --- a/backend/.sqlx/query-a59b70164dc87224d09a04d5469ca217eb19a15a250c3b83ca63f606f89b9681.json +++ b/backend/.sqlx/query-08c1121171b98889f188ea6b33b1861f3483fa70b5d58dd2838a5cb6dabe9cc1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()", + "query": "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "a59b70164dc87224d09a04d5469ca217eb19a15a250c3b83ca63f606f89b9681" + "hash": "08c1121171b98889f188ea6b33b1861f3483fa70b5d58dd2838a5cb6dabe9cc1" } diff --git a/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json b/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json index fac1d666f7..d266d7db3c 100644 --- a/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json +++ b/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json @@ -157,6 +157,16 @@ "ordinal": 30, "name": "auto_add_instance_groups_roles", "type_info": "Jsonb" + }, + { + "ordinal": 31, + "name": "slack_oauth_client_id", + "type_info": "Varchar" + }, + { + "ordinal": 32, + "name": "slack_oauth_client_secret", + "type_info": "Varchar" } ], "parameters": { @@ -195,6 +205,8 @@ false, true, true, + true, + true, true ] }, diff --git a/backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json b/backend/.sqlx/query-0a7132202ecf6c4c10340921644a90d9206c45d92a0423c0bc2396d0d66a0b0d.json similarity index 84% rename from backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json rename to backend/.sqlx/query-0a7132202ecf6c4c10340921644a90d9206c45d92a0423c0bc2396d0d66a0b0d.json index 9462f9b48b..ecaf828737 100644 --- a/backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json +++ b/backend/.sqlx/query-0a7132202ecf6c4c10340921644a90d9206c45d92a0423c0bc2396d0d66a0b0d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34)", + "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36)", "describe": { "columns": [], "parameters": { @@ -83,10 +83,12 @@ "Bool", "Text", "Bool", - "Jsonb" + "Jsonb", + "Varchar", + "Int4" ] }, "nullable": [] }, - "hash": "1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e" + "hash": "0a7132202ecf6c4c10340921644a90d9206c45d92a0423c0bc2396d0d66a0b0d" } diff --git a/backend/.sqlx/query-0ae9160591ae00117d20a616cfe07e38f0c32953c7e881e916c389255190b72d.json b/backend/.sqlx/query-0ae9160591ae00117d20a616cfe07e38f0c32953c7e881e916c389255190b72d.json deleted file mode 100644 index aae4302601..0000000000 --- a/backend/.sqlx/query-0ae9160591ae00117d20a616cfe07e38f0c32953c7e881e916c389255190b72d.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usr\n (workspace_id, email, username, is_admin)\n VALUES ($1, $2, $3, $4)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "0ae9160591ae00117d20a616cfe07e38f0c32953c7e881e916c389255190b72d" -} diff --git a/backend/.sqlx/query-0b43d1f0c0d205d978cdb41d30835a6a41a13f39159e106834c62f3b46c44227.json b/backend/.sqlx/query-0b43d1f0c0d205d978cdb41d30835a6a41a13f39159e106834c62f3b46c44227.json new file mode 100644 index 0000000000..7091c464ad --- /dev/null +++ b/backend/.sqlx/query-0b43d1f0c0d205d978cdb41d30835a6a41a13f39159e106834c62f3b46c44227.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT usage FROM usage\n WHERE id = $1\n AND is_workspace = FALSE\n AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "usage", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0b43d1f0c0d205d978cdb41d30835a6a41a13f39159e106834c62f3b46c44227" +} diff --git a/backend/.sqlx/query-0bcbed8d2a7ad88b809a211a8c13a3d74b8e8141be95cbcd63e227d13091a8dd.json b/backend/.sqlx/query-0bcbed8d2a7ad88b809a211a8c13a3d74b8e8141be95cbcd63e227d13091a8dd.json new file mode 100644 index 0000000000..fa2b39ff0b --- /dev/null +++ b/backend/.sqlx/query-0bcbed8d2a7ad88b809a211a8c13a3d74b8e8141be95cbcd63e227d13091a8dd.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, hash FROM script WHERE workspace_id = $1 AND archived = false AND deleted = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "hash", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "0bcbed8d2a7ad88b809a211a8c13a3d74b8e8141be95cbcd63e227d13091a8dd" +} diff --git a/backend/.sqlx/query-0cc221cb8b3059b21e6b3b4c874b8f4d32815edd2090ccb5d562a89142a7dd9c.json b/backend/.sqlx/query-0cc221cb8b3059b21e6b3b4c874b8f4d32815edd2090ccb5d562a89142a7dd9c.json new file mode 100644 index 0000000000..fbe7e2dec1 --- /dev/null +++ b/backend/.sqlx/query-0cc221cb8b3059b21e6b3b4c874b8f4d32815edd2090ccb5d562a89142a7dd9c.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "VACUUM (SKIP_LOCKED) v2_job_queue, v2_job_runtime, v2_job_status, job_perms", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "0cc221cb8b3059b21e6b3b4c874b8f4d32815edd2090ccb5d562a89142a7dd9c" +} diff --git a/backend/.sqlx/query-1060c503cf8d4bb5cef9720c162b8192924b4a938d249fae92624cd55e44f488.json b/backend/.sqlx/query-1060c503cf8d4bb5cef9720c162b8192924b4a938d249fae92624cd55e44f488.json deleted file mode 100644 index 39b7179e5c..0000000000 --- a/backend/.sqlx/query-1060c503cf8d4bb5cef9720c162b8192924b4a938d249fae92624cd55e44f488.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n flow_status,\n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT],\n $4\n )\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "1060c503cf8d4bb5cef9720c162b8192924b4a938d249fae92624cd55e44f488" -} diff --git a/backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json b/backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json deleted file mode 100644 index 66a2152ae9..0000000000 --- a/backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n mqtt_trigger \n SET \n server_id = $1, \n last_server_ping = now(),\n error = 'Connecting...'\n WHERE \n enabled IS TRUE \n AND workspace_id = $2 \n AND path = $3 \n AND (last_server_ping IS NULL \n OR last_server_ping < now() - INTERVAL '15 seconds'\n ) \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a" -} diff --git a/backend/.sqlx/query-12d37d75a429c0ddf2b2c190ab28bea5aefd27d0ed8a1bb2c8b3c1b0ece4efb7.json b/backend/.sqlx/query-12d37d75a429c0ddf2b2c190ab28bea5aefd27d0ed8a1bb2c8b3c1b0ece4efb7.json new file mode 100644 index 0000000000..684f4a6cb4 --- /dev/null +++ b/backend/.sqlx/query-12d37d75a429c0ddf2b2c190ab28bea5aefd27d0ed8a1bb2c8b3c1b0ece4efb7.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH to_update AS (\n SELECT q.id, q.workspace_id, r.ping, COALESCE(zjc.counter, 0) as counter\n FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n JOIN v2_job_runtime r ON r.id = j.id\n LEFT JOIN zombie_job_counter zjc ON zjc.job_id = q.id\n WHERE ping < now() - ($1 || ' seconds')::interval\n AND running = true\n AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND same_worker = false\n AND (zjc.counter IS NULL OR zjc.counter <= $2)\n FOR UPDATE of q SKIP LOCKED\n ),\n zombie_jobs AS (\n UPDATE v2_job_queue q\n SET running = false, started_at = null\n FROM to_update tu\n WHERE q.id = tu.id AND (tu.counter IS NULL OR tu.counter < $2)\n RETURNING q.id, q.workspace_id, ping, tu.counter\n ),\n update_ping AS (\n UPDATE v2_job_runtime r\n SET ping = null\n FROM zombie_jobs zj\n WHERE r.id = zj.id\n ),\n increment_counter AS (\n INSERT INTO zombie_job_counter (job_id, counter)\n SELECT id, 1 FROM to_update WHERE counter < $2\n ON CONFLICT (job_id) DO UPDATE\n SET counter = zombie_job_counter.counter + 1\n ),\n update_concurrency AS (\n UPDATE concurrency_counter cc\n SET job_uuids = job_uuids - zj.id::text\n FROM zombie_jobs zj\n INNER JOIN concurrency_key ck ON ck.job_id = zj.id\n WHERE cc.concurrency_id = ck.key\n )\n SELECT id AS \"id!\", workspace_id AS \"workspace_id!\", ping, counter + 1 AS counter FROM to_update", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "counter", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Int4" + ] + }, + "nullable": [ + false, + false, + true, + null + ] + }, + "hash": "12d37d75a429c0ddf2b2c190ab28bea5aefd27d0ed8a1bb2c8b3c1b0ece4efb7" +} diff --git a/backend/.sqlx/query-12e868b63a7c622c76713db5a5577a927efca4ae49a15c2b999e2410f2a312ff.json b/backend/.sqlx/query-12e868b63a7c622c76713db5a5577a927efca4ae49a15c2b999e2410f2a312ff.json deleted file mode 100644 index 49ad10ac07..0000000000 --- a/backend/.sqlx/query-12e868b63a7c622c76713db5a5577a927efca4ae49a15c2b999e2410f2a312ff.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE \n capture_config \n SET \n last_server_ping = NULL \n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = 'postgres' AND \n server_id IS NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "12e868b63a7c622c76713db5a5577a927efca4ae49a15c2b999e2410f2a312ff" -} diff --git a/backend/.sqlx/query-4410b2d6e52556569cda9d945f791756a80ff2835d74bfab85f53141b6e32351.json b/backend/.sqlx/query-1301f873a829db137573b8b39449f6160f2adf44f864f26a99b8eab5818fbd50.json similarity index 66% rename from backend/.sqlx/query-4410b2d6e52556569cda9d945f791756a80ff2835d74bfab85f53141b6e32351.json rename to backend/.sqlx/query-1301f873a829db137573b8b39449f6160f2adf44f864f26a99b8eab5818fbd50.json index 63c326f5e2..2999ae4bf2 100644 --- a/backend/.sqlx/query-4410b2d6e52556569cda9d945f791756a80ff2835d74bfab85f53141b6e32351.json +++ b/backend/.sqlx/query-1301f873a829db137573b8b39449f6160f2adf44f864f26a99b8eab5818fbd50.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n path, \n script_path, \n is_flow, \n route_path, \n authentication_resource_path,\n workspace_id, \n is_async, \n authentication_method AS \"authentication_method: _\", \n edited_by, \n email, \n static_asset_config AS \"static_asset_config: _\",\n wrap_body,\n raw_string,\n workspaced_route,\n is_static_website,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM \n http_trigger \n WHERE \n http_method = $1\n ", + "query": "\n SELECT\n path,\n script_path,\n is_flow,\n route_path,\n authentication_resource_path,\n workspace_id,\n request_type AS \"request_type: _\",\n authentication_method AS \"authentication_method: _\",\n edited_by,\n email,\n static_asset_config AS \"static_asset_config: _\",\n wrap_body,\n raw_string,\n workspaced_route,\n is_static_website,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM\n http_trigger\n WHERE\n http_method = $1\n ", "describe": { "columns": [ { @@ -35,8 +35,19 @@ }, { "ordinal": 6, - "name": "is_async", - "type_info": "Bool" + "name": "request_type: _", + "type_info": { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + } }, { "ordinal": 7, @@ -147,5 +158,5 @@ true ] }, - "hash": "4410b2d6e52556569cda9d945f791756a80ff2835d74bfab85f53141b6e32351" + "hash": "1301f873a829db137573b8b39449f6160f2adf44f864f26a99b8eab5818fbd50" } diff --git a/backend/.sqlx/query-3700706bb0408d6593be7f15f8aee4d6a023109e5dcfee7009a25ddf5db5e28d.json b/backend/.sqlx/query-13297889361ac6839d6c4bd0b8ae121305d63cfcd88f69700125d17fb2c56a1f.json similarity index 53% rename from backend/.sqlx/query-3700706bb0408d6593be7f15f8aee4d6a023109e5dcfee7009a25ddf5db5e28d.json rename to backend/.sqlx/query-13297889361ac6839d6c4bd0b8ae121305d63cfcd88f69700125d17fb2c56a1f.json index 8ccffb9455..3a17f37f37 100644 --- a/backend/.sqlx/query-3700706bb0408d6593be7f15f8aee4d6a023109e5dcfee7009a25ddf5db5e28d.json +++ b/backend/.sqlx/query-13297889361ac6839d6c4bd0b8ae121305d63cfcd88f69700125d17fb2c56a1f.json @@ -1,36 +1,31 @@ { "db_name": "PostgreSQL", - "query": "SELECT script_path, is_flow, workspace_id, edited_by, email, path FROM email_trigger WHERE local_part = $1 AND workspaced_local_part = FALSE", + "query": "\n SELECT workspace_id, importer_path, importer_kind::text, imported_path, importer_node_id\n FROM dependency_map WHERE workspace_id = $1", "describe": { "columns": [ { "ordinal": 0, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, "name": "workspace_id", "type_info": "Varchar" }, + { + "ordinal": 1, + "name": "importer_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "importer_kind", + "type_info": "Text" + }, { "ordinal": 3, - "name": "edited_by", + "name": "imported_path", "type_info": "Varchar" }, { "ordinal": 4, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "path", + "name": "importer_node_id", "type_info": "Varchar" } ], @@ -42,11 +37,10 @@ "nullable": [ false, false, - false, - false, + null, false, false ] }, - "hash": "3700706bb0408d6593be7f15f8aee4d6a023109e5dcfee7009a25ddf5db5e28d" + "hash": "13297889361ac6839d6c4bd0b8ae121305d63cfcd88f69700125d17fb2c56a1f" } diff --git a/backend/.sqlx/query-6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c.json b/backend/.sqlx/query-1368ccd2c15a75690041a6c87d4a2849fe6bc668654ffcbfbc22a02027280739.json similarity index 63% rename from backend/.sqlx/query-6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c.json rename to backend/.sqlx/query-1368ccd2c15a75690041a6c87d4a2849fe6bc668654ffcbfbc22a02027280739.json index ef26a8453e..4dac49a78c 100644 --- a/backend/.sqlx/query-6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c.json +++ b/backend/.sqlx/query-1368ccd2c15a75690041a6c87d4a2849fe6bc668654ffcbfbc22a02027280739.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COUNT(id) FROM v2_as_queue WHERE email = $1", + "query": "SELECT COUNT(id) FROM v2_job WHERE permissioned_as_email = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c" + "hash": "1368ccd2c15a75690041a6c87d4a2849fe6bc668654ffcbfbc22a02027280739" } diff --git a/backend/.sqlx/query-140e77db6b38574c62f35d23b876e749d2c43af837f2b3f3edbf49f979e44082.json b/backend/.sqlx/query-140e77db6b38574c62f35d23b876e749d2c43af837f2b3f3edbf49f979e44082.json new file mode 100644 index 0000000000..1734a545fe --- /dev/null +++ b/backend/.sqlx/query-140e77db6b38574c62f35d23b876e749d2c43af837f2b3f3edbf49f979e44082.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT flow_status->'modules'->$2::int->'flow_jobs_success' as \"flow_jobs_success: Json>>\", flow_status->'modules'->$2::int->'flow_jobs_duration' as \"flow_jobs_duration: Json\"\n FROM v2_job_status WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flow_jobs_success: Json>>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "flow_jobs_duration: Json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Int4" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "140e77db6b38574c62f35d23b876e749d2c43af837f2b3f3edbf49f979e44082" +} diff --git a/backend/.sqlx/query-145b364bcd45b6a8b3b80fd67a5ae17212785bd7206fbd3901a3b516eb77dc55.json b/backend/.sqlx/query-145b364bcd45b6a8b3b80fd67a5ae17212785bd7206fbd3901a3b516eb77dc55.json new file mode 100644 index 0000000000..574f239135 --- /dev/null +++ b/backend/.sqlx/query-145b364bcd45b6a8b3b80fd67a5ae17212785bd7206fbd3901a3b516eb77dc55.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(\n SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2 AND archived = false AND deleted = false\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "145b364bcd45b6a8b3b80fd67a5ae17212785bd7206fbd3901a3b516eb77dc55" +} diff --git a/backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json b/backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json deleted file mode 100644 index 197298f788..0000000000 --- a/backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE postgres_trigger \n SET \n server_id = $1, \n last_server_ping = now(),\n error = 'Connecting...'\n WHERE \n enabled IS TRUE \n AND workspace_id = $2 \n AND path = $3 \n AND (last_server_ping IS NULL \n OR last_server_ping < now() - INTERVAL '15 seconds'\n ) \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416" -} diff --git a/backend/.sqlx/query-1492b88c75722465b1a5c138729e6bb2782e1f8ef5f9fe752b356927a8605100.json b/backend/.sqlx/query-1492b88c75722465b1a5c138729e6bb2782e1f8ef5f9fe752b356927a8605100.json new file mode 100644 index 0000000000..d5b29d83dd --- /dev/null +++ b/backend/.sqlx/query-1492b88c75722465b1a5c138729e6bb2782e1f8ef5f9fe752b356927a8605100.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM app_version WHERE id = $1 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1492b88c75722465b1a5c138729e6bb2782e1f8ef5f9fe752b356927a8605100" +} diff --git a/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json similarity index 90% rename from backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json rename to backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json index 0051151fc0..918967e934 100644 --- a/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json +++ b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT * FROM workspace_settings WHERE slack_team_id = $1 AND slack_command_script IS NOT NULL", + "query": "SELECT * FROM workspace_settings WHERE workspace_id = $1", "describe": { "columns": [ { @@ -157,6 +157,16 @@ "ordinal": 30, "name": "auto_add_instance_groups_roles", "type_info": "Jsonb" + }, + { + "ordinal": 31, + "name": "slack_oauth_client_id", + "type_info": "Varchar" + }, + { + "ordinal": 32, + "name": "slack_oauth_client_secret", + "type_info": "Varchar" } ], "parameters": { @@ -195,8 +205,10 @@ false, true, true, + true, + true, true ] }, - "hash": "55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2" + "hash": "1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597" } diff --git a/backend/.sqlx/query-1a9ba16c90d3d65c4ff39aaddb3079009e03af711e7f6b53332537cf4cb0e8dd.json b/backend/.sqlx/query-17ca259e1c78e1317fdd19436e15bef428fc4f0d52776d7a5fca64f17225ef30.json similarity index 83% rename from backend/.sqlx/query-1a9ba16c90d3d65c4ff39aaddb3079009e03af711e7f6b53332537cf4cb0e8dd.json rename to backend/.sqlx/query-17ca259e1c78e1317fdd19436e15bef428fc4f0d52776d7a5fca64f17225ef30.json index 141f92e271..dd40aabbc1 100644 --- a/backend/.sqlx/query-1a9ba16c90d3d65c4ff39aaddb3079009e03af711e7f6b53332537cf4cb0e8dd.json +++ b/backend/.sqlx/query-17ca259e1c78e1317fdd19436e15bef428fc4f0d52776d7a5fca64f17225ef30.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE \n gcp_trigger \n SET \n gcp_resource_path = $1,\n subscription_id = $2,\n topic_id = $3,\n delivery_type = $4,\n delivery_config = $5,\n is_flow = $6, \n edited_by = $7, \n email = $8,\n script_path = $9,\n path = $10,\n enabled = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16,\n auto_acknowledge_msg = $17\n WHERE \n workspace_id = $12 AND \n path = $13\n ", + "query": "\n UPDATE \n gcp_trigger \n SET \n gcp_resource_path = $1,\n subscription_id = $2,\n topic_id = $3,\n delivery_type = $4,\n delivery_config = $5,\n is_flow = $6, \n edited_by = $7, \n email = $8,\n script_path = $9,\n path = $10,\n enabled = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16,\n auto_acknowledge_msg = $17,\n ack_deadline = $18\n WHERE \n workspace_id = $12 AND \n path = $13\n ", "describe": { "columns": [], "parameters": { @@ -31,10 +31,11 @@ "Varchar", "Jsonb", "Jsonb", - "Bool" + "Bool", + "Int4" ] }, "nullable": [] }, - "hash": "1a9ba16c90d3d65c4ff39aaddb3079009e03af711e7f6b53332537cf4cb0e8dd" + "hash": "17ca259e1c78e1317fdd19436e15bef428fc4f0d52776d7a5fca64f17225ef30" } diff --git a/backend/.sqlx/query-186aef850c2eeb89c186ac6b2934dd3a703e2b9428096801e1d2d61fdbb99c9e.json b/backend/.sqlx/query-186aef850c2eeb89c186ac6b2934dd3a703e2b9428096801e1d2d61fdbb99c9e.json deleted file mode 100644 index 648c5dda19..0000000000 --- a/backend/.sqlx/query-186aef850c2eeb89c186ac6b2934dd3a703e2b9428096801e1d2d61fdbb99c9e.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n mqtt_trigger \n SET\n last_server_ping = NULL \n WHERE \n workspace_id = $1 \n AND path = $2 \n AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "186aef850c2eeb89c186ac6b2934dd3a703e2b9428096801e1d2d61fdbb99c9e" -} diff --git a/backend/.sqlx/query-1974bd65bbf40024773aad4dee1c50b12e110e76bb58e6de25bec094e758a71c.json b/backend/.sqlx/query-1974bd65bbf40024773aad4dee1c50b12e110e76bb58e6de25bec094e758a71c.json deleted file mode 100644 index f19670704d..0000000000 --- a/backend/.sqlx/query-1974bd65bbf40024773aad4dee1c50b12e110e76bb58e6de25bec094e758a71c.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n last_server_ping = now(), \n error = $1 \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'postgres' AND \n server_id = $5 AND \n last_client_ping > NOW() - INTERVAL '10 seconds' \n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "1974bd65bbf40024773aad4dee1c50b12e110e76bb58e6de25bec094e758a71c" -} diff --git a/backend/.sqlx/query-19b3b850912ea52503be2a3bfbdee2f0d472328d5be918a76098108d76be6710.json b/backend/.sqlx/query-19b3b850912ea52503be2a3bfbdee2f0d472328d5be918a76098108d76be6710.json new file mode 100644 index 0000000000..1d33abb64d --- /dev/null +++ b/backend/.sqlx/query-19b3b850912ea52503be2a3bfbdee2f0d472328d5be918a76098108d76be6710.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT slack_oauth_client_id, slack_oauth_client_secret FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "slack_oauth_client_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "slack_oauth_client_secret", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "19b3b850912ea52503be2a3bfbdee2f0d472328d5be918a76098108d76be6710" +} diff --git a/backend/.sqlx/query-8bd028c8b5f8a4d566f89eebc2e63fd04beaf2b0b49e07c7df42ecddd70737f3.json b/backend/.sqlx/query-19e4625de06b8bab10039280a6213df5f38fb8892226f04cf700f60eb45199ef.json similarity index 75% rename from backend/.sqlx/query-8bd028c8b5f8a4d566f89eebc2e63fd04beaf2b0b49e07c7df42ecddd70737f3.json rename to backend/.sqlx/query-19e4625de06b8bab10039280a6213df5f38fb8892226f04cf700f60eb45199ef.json index f1f7a0b56c..bec2b14783 100644 --- a/backend/.sqlx/query-8bd028c8b5f8a4d566f89eebc2e63fd04beaf2b0b49e07c7df42ecddd70737f3.json +++ b/backend/.sqlx/query-19e4625de06b8bab10039280a6213df5f38fb8892226f04cf700f60eb45199ef.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO resource\n (workspace_id, path, value, resource_type, created_by, edited_at)\n VALUES ($1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = $3, edited_at = now()", + "query": "INSERT INTO resource\n (workspace_id, path, value, resource_type, created_by, edited_at)\n VALUES ($1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = EXCLUDED.value, edited_at = now()", "describe": { "columns": [], "parameters": { @@ -14,5 +14,5 @@ }, "nullable": [] }, - "hash": "8bd028c8b5f8a4d566f89eebc2e63fd04beaf2b0b49e07c7df42ecddd70737f3" + "hash": "19e4625de06b8bab10039280a6213df5f38fb8892226f04cf700f60eb45199ef" } diff --git a/backend/.sqlx/query-1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078.json b/backend/.sqlx/query-1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078.json new file mode 100644 index 0000000000..0265d7d2b5 --- /dev/null +++ b/backend/.sqlx/query-1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr\n (workspace_id, email, username, is_admin)\n SELECT $1, email, username, is_admin FROM usr\n WHERE workspace_id = $3 AND email = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078" +} diff --git a/backend/.sqlx/query-1ace9bdcde90fd2261fd64344a2d9474b17887711128dbb2ef15d247d50686b0.json b/backend/.sqlx/query-1ace9bdcde90fd2261fd64344a2d9474b17887711128dbb2ef15d247d50686b0.json deleted file mode 100644 index 9a8ef973a4..0000000000 --- a/backend/.sqlx/query-1ace9bdcde90fd2261fd64344a2d9474b17887711128dbb2ef15d247d50686b0.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT teams_team_id FROM workspace_settings WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "teams_team_id", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - true - ] - }, - "hash": "1ace9bdcde90fd2261fd64344a2d9474b17887711128dbb2ef15d247d50686b0" -} diff --git a/backend/.sqlx/query-1b33393fbbc7e681b4d355f6096d982b52b48c8c3da392b14963a1ec86811546.json b/backend/.sqlx/query-1b33393fbbc7e681b4d355f6096d982b52b48c8c3da392b14963a1ec86811546.json deleted file mode 100644 index 271c60b480..0000000000 --- a/backend/.sqlx/query-1b33393fbbc7e681b4d355f6096d982b52b48c8c3da392b14963a1ec86811546.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n path,\n is_flow,\n workspace_id,\n owner,\n email,\n trigger_config as \"trigger_config!: _\"\n FROM\n capture_config\n WHERE\n trigger_kind = 'sqs' AND\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n trigger_config IS NOT NULL AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "owner", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "trigger_config!: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - false, - true - ] - }, - "hash": "1b33393fbbc7e681b4d355f6096d982b52b48c8c3da392b14963a1ec86811546" -} diff --git a/backend/.sqlx/query-1c37f91192aa4f535c7fff80fa809260b906dc988f44a0db60952a2bf8b1cdaf.json b/backend/.sqlx/query-1c37f91192aa4f535c7fff80fa809260b906dc988f44a0db60952a2bf8b1cdaf.json deleted file mode 100644 index 946783d745..0000000000 --- a/backend/.sqlx/query-1c37f91192aa4f535c7fff80fa809260b906dc988f44a0db60952a2bf8b1cdaf.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE kafka_trigger SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "1c37f91192aa4f535c7fff80fa809260b906dc988f44a0db60952a2bf8b1cdaf" -} diff --git a/backend/.sqlx/query-1cad25c24d0f80d58a50d4da923f8b0672797299545ee0754b6ad74ece92c77e.json b/backend/.sqlx/query-1cad25c24d0f80d58a50d4da923f8b0672797299545ee0754b6ad74ece92c77e.json deleted file mode 100644 index 0729adc8ba..0000000000 --- a/backend/.sqlx/query-1cad25c24d0f80d58a50d4da923f8b0672797299545ee0754b6ad74ece92c77e.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n last_server_ping = now(), \n error = $1 \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'sqs' AND \n server_id = $5 AND \n last_client_ping > NOW() - INTERVAL '10 seconds' \n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "1cad25c24d0f80d58a50d4da923f8b0672797299545ee0754b6ad74ece92c77e" -} diff --git a/backend/.sqlx/query-1d9498226b3d962688558d8ab77f88de5dc8ae5321fa7ff0c7632e628556f991.json b/backend/.sqlx/query-1d9498226b3d962688558d8ab77f88de5dc8ae5321fa7ff0c7632e628556f991.json new file mode 100644 index 0000000000..4e4ae9586c --- /dev/null +++ b/backend/.sqlx/query-1d9498226b3d962688558d8ab77f88de5dc8ae5321fa7ff0c7632e628556f991.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2 RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1d9498226b3d962688558d8ab77f88de5dc8ae5321fa7ff0c7632e628556f991" +} diff --git a/backend/.sqlx/query-9f1f388924176dbe3dea882e0c62728a82ba256029096812dd705ccb1a552cfe.json b/backend/.sqlx/query-1e426c8a06d7bbed7af67a105f74b3e03bd44048af4d69fab854f97fa821649b.json similarity index 78% rename from backend/.sqlx/query-9f1f388924176dbe3dea882e0c62728a82ba256029096812dd705ccb1a552cfe.json rename to backend/.sqlx/query-1e426c8a06d7bbed7af67a105f74b3e03bd44048af4d69fab854f97fa821649b.json index 51ddd8af63..0f4f6a0a90 100644 --- a/backend/.sqlx/query-9f1f388924176dbe3dea882e0c62728a82ba256029096812dd705ccb1a552cfe.json +++ b/backend/.sqlx/query-1e426c8a06d7bbed7af67a105f74b3e03bd44048af4d69fab854f97fa821649b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO variable\n (workspace_id, path, value, is_secret, account, is_oauth)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (workspace_id, path) DO UPDATE SET value = $3", + "query": "INSERT INTO variable\n (workspace_id, path, value, is_secret, account, is_oauth)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (workspace_id, path) DO UPDATE SET value = EXCLUDED.value", "describe": { "columns": [], "parameters": { @@ -15,5 +15,5 @@ }, "nullable": [] }, - "hash": "9f1f388924176dbe3dea882e0c62728a82ba256029096812dd705ccb1a552cfe" + "hash": "1e426c8a06d7bbed7af67a105f74b3e03bd44048af4d69fab854f97fa821649b" } diff --git a/backend/.sqlx/query-1f6b773ce34fe51d03d6f9a2345481629c62453eebbb08f82dd2da23389bc117.json b/backend/.sqlx/query-1f6b773ce34fe51d03d6f9a2345481629c62453eebbb08f82dd2da23389bc117.json new file mode 100644 index 0000000000..f69ec4f2a6 --- /dev/null +++ b/backend/.sqlx/query-1f6b773ce34fe51d03d6f9a2345481629c62453eebbb08f82dd2da23389bc117.json @@ -0,0 +1,73 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO http_trigger (\n workspace_id,\n path,\n route_path,\n route_path_key,\n workspaced_route,\n authentication_resource_path,\n wrap_body,\n raw_string,\n script_path,\n summary,\n description,\n is_flow,\n request_type,\n authentication_method,\n http_method,\n static_asset_config,\n edited_by,\n email,\n edited_at,\n is_static_website,\n error_handler_path,\n error_handler_args,\n retry\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, now(), $19, $20, $21, $22\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Bool", + "Bool", + "Varchar", + "Varchar", + "Text", + "Bool", + { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + }, + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Jsonb", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "1f6b773ce34fe51d03d6f9a2345481629c62453eebbb08f82dd2da23389bc117" +} diff --git a/backend/.sqlx/query-1ff185d8b8b897a72180cd0002e0f7e9858eef249577ce23969522793c6b5608.json b/backend/.sqlx/query-1ff185d8b8b897a72180cd0002e0f7e9858eef249577ce23969522793c6b5608.json new file mode 100644 index 0000000000..6b8589e8e6 --- /dev/null +++ b/backend/.sqlx/query-1ff185d8b8b897a72180cd0002e0f7e9858eef249577ce23969522793c6b5608.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n eig.igroup as group_name,\n ws.auto_add_instance_groups_roles\n FROM email_to_igroup eig\n INNER JOIN workspace_settings ws ON ws.workspace_id = $1\n WHERE eig.email = $2\n AND eig.igroup = ANY(ws.auto_add_instance_groups)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "group_name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "auto_add_instance_groups_roles", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "1ff185d8b8b897a72180cd0002e0f7e9858eef249577ce23969522793c6b5608" +} diff --git a/backend/.sqlx/query-203fa78d423ec5a8c5ff6166aed591b28cbf9ea8f61d379b84ee6e14c033035d.json b/backend/.sqlx/query-203fa78d423ec5a8c5ff6166aed591b28cbf9ea8f61d379b84ee6e14c033035d.json deleted file mode 100644 index 65b1b24efb..0000000000 --- a/backend/.sqlx/query-203fa78d423ec5a8c5ff6166aed591b28cbf9ea8f61d379b84ee6e14c033035d.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'kafka'", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "203fa78d423ec5a8c5ff6166aed591b28cbf9ea8f61d379b84ee6e14c033035d" -} diff --git a/backend/.sqlx/query-20e01ecb5d4aa4c532a8f906365c776994ae2c990bb7bb307c52296b9543fbb8.json b/backend/.sqlx/query-20e01ecb5d4aa4c532a8f906365c776994ae2c990bb7bb307c52296b9543fbb8.json deleted file mode 100644 index c9f7733011..0000000000 --- a/backend/.sqlx/query-20e01ecb5d4aa4c532a8f906365c776994ae2c990bb7bb307c52296b9543fbb8.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE \n capture_config \n SET \n last_server_ping = NULL \n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = 'sqs' AND \n server_id IS NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "20e01ecb5d4aa4c532a8f906365c776994ae2c990bb7bb307c52296b9543fbb8" -} diff --git a/backend/.sqlx/query-22dcd953d900fb0ddbe2099ccee27bcf833f56475bbc1aa3c8f6d79f146f41bf.json b/backend/.sqlx/query-22dcd953d900fb0ddbe2099ccee27bcf833f56475bbc1aa3c8f6d79f146f41bf.json deleted file mode 100644 index f94ca24f0f..0000000000 --- a/backend/.sqlx/query-22dcd953d900fb0ddbe2099ccee27bcf833f56475bbc1aa3c8f6d79f146f41bf.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n sqs_trigger\n SET \n last_server_ping = now(),\n error = $1\n WHERE\n workspace_id = $2\n AND path = $3\n AND server_id = $4 \n AND enabled IS TRUE\n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "22dcd953d900fb0ddbe2099ccee27bcf833f56475bbc1aa3c8f6d79f146f41bf" -} diff --git a/backend/.sqlx/query-a0c20436e0506bf9e0e50bd3dcabdd35131cc374edac0a08c5492954f08c9d98.json b/backend/.sqlx/query-23e4c6e3dc6a48f702c2b26a6b1f94668e086caaa0093a3b685f87483513b0d2.json similarity index 74% rename from backend/.sqlx/query-a0c20436e0506bf9e0e50bd3dcabdd35131cc374edac0a08c5492954f08c9d98.json rename to backend/.sqlx/query-23e4c6e3dc6a48f702c2b26a6b1f94668e086caaa0093a3b685f87483513b0d2.json index 1724d969cf..fe2b1e3427 100644 --- a/backend/.sqlx/query-a0c20436e0506bf9e0e50bd3dcabdd35131cc374edac0a08c5492954f08c9d98.json +++ b/backend/.sqlx/query-23e4c6e3dc6a48f702c2b26a6b1f94668e086caaa0093a3b685f87483513b0d2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO schedule (\n workspace_id, path, schedule, timezone, edited_by, script_path,\n is_flow, args, enabled, email,\n on_failure, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery, on_recovery_times, on_recovery_extra_args,\n on_success, on_success_extra_args,\n ws_error_handler_muted, retry, summary, no_flow_overlap,\n tag, paused_until, cron_version, description\n ) VALUES (\n $1, $2, $3, $4, $5, $6,\n $7, $8, $9, $10,\n $11, $12, $13, $14,\n $15, $16, $17,\n $18, $19,\n $20, $21, $22, $23,\n $24, $25, $26, $27\n )\n RETURNING\n workspace_id,\n path,\n edited_by,\n edited_at,\n schedule,\n timezone,\n enabled,\n script_path,\n is_flow,\n args AS \"args: _\",\n extra_perms,\n email,\n error,\n on_failure,\n on_failure_times,\n on_failure_exact,\n on_failure_extra_args AS \"on_failure_extra_args: _\",\n on_recovery,\n on_recovery_times,\n on_recovery_extra_args AS \"on_recovery_extra_args: _\",\n on_success,\n on_success_extra_args AS \"on_success_extra_args: _\",\n ws_error_handler_muted,\n retry,\n no_flow_overlap,\n summary,\n description,\n tag,\n paused_until,\n cron_version\n ", + "query": "\n INSERT INTO schedule (\n workspace_id, path, schedule, timezone, edited_by, script_path,\n is_flow, args, enabled, email,\n on_failure, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery, on_recovery_times, on_recovery_extra_args,\n on_success, on_success_extra_args,\n ws_error_handler_muted, retry, summary, no_flow_overlap,\n tag, paused_until, cron_version, description, dynamic_skip\n ) VALUES (\n $1, $2, $3, $4, $5, $6,\n $7, $8, $9, $10,\n $11, $12, $13, $14,\n $15, $16, $17,\n $18, $19,\n $20, $21, $22, $23,\n $24, $25, $26, $27, $28\n )\n RETURNING\n workspace_id,\n path,\n edited_by,\n edited_at,\n schedule,\n timezone,\n enabled,\n script_path,\n is_flow,\n args AS \"args: _\",\n extra_perms,\n email,\n error,\n on_failure,\n on_failure_times,\n on_failure_exact,\n on_failure_extra_args AS \"on_failure_extra_args: _\",\n on_recovery,\n on_recovery_times,\n on_recovery_extra_args AS \"on_recovery_extra_args: _\",\n on_success,\n on_success_extra_args AS \"on_success_extra_args: _\",\n ws_error_handler_muted,\n retry,\n no_flow_overlap,\n summary,\n description,\n tag,\n paused_until,\n cron_version,\n dynamic_skip\n ", "describe": { "columns": [ { @@ -152,6 +152,11 @@ "ordinal": 29, "name": "cron_version", "type_info": "Text" + }, + { + "ordinal": 30, + "name": "dynamic_skip", + "type_info": "Varchar" } ], "parameters": { @@ -182,7 +187,8 @@ "Varchar", "Timestamptz", "Text", - "Text" + "Text", + "Varchar" ] }, "nullable": [ @@ -215,8 +221,9 @@ true, true, true, + true, true ] }, - "hash": "a0c20436e0506bf9e0e50bd3dcabdd35131cc374edac0a08c5492954f08c9d98" + "hash": "23e4c6e3dc6a48f702c2b26a6b1f94668e086caaa0093a3b685f87483513b0d2" } diff --git a/backend/.sqlx/query-acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359.json b/backend/.sqlx/query-24d302b8215d49a289bedd14a5791e9366d1f6d3d2aa485e0f50c6f2d85693dd.json similarity index 52% rename from backend/.sqlx/query-acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359.json rename to backend/.sqlx/query-24d302b8215d49a289bedd14a5791e9366d1f6d3d2aa485e0f50c6f2d85693dd.json index 58f693df33..7acbf9deec 100644 --- a/backend/.sqlx/query-acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359.json +++ b/backend/.sqlx/query-24d302b8215d49a289bedd14a5791e9366d1f6d3d2aa485e0f50c6f2d85693dd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id AS \"id!\" FROM v2_as_queue WHERE id = ANY($1) AND schedule_path IS NULL AND ($2::text[] IS NULL OR tag = ANY($2))", + "query": "SELECT j.id AS \"id!\" FROM v2_job j WHERE j.id = ANY($1) AND j.trigger_kind != 'schedule'::job_trigger_kind AND ($2::text[] IS NULL OR j.tag = ANY($2))", "describe": { "columns": [ { @@ -16,8 +16,8 @@ ] }, "nullable": [ - true + false ] }, - "hash": "acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359" + "hash": "24d302b8215d49a289bedd14a5791e9366d1f6d3d2aa485e0f50c6f2d85693dd" } diff --git a/backend/.sqlx/query-a00e61e770e20157bbd9e4cdedf7fb5f9de7c8c9e50282e3ecf2e3ce917ec37a.json b/backend/.sqlx/query-24f38f0642b49626c8c8417e1846ab38dfe15284a6a7f54366ba25d7eb75a74a.json similarity index 62% rename from backend/.sqlx/query-a00e61e770e20157bbd9e4cdedf7fb5f9de7c8c9e50282e3ecf2e3ce917ec37a.json rename to backend/.sqlx/query-24f38f0642b49626c8c8417e1846ab38dfe15284a6a7f54366ba25d7eb75a74a.json index 9f95781136..265ac4fba1 100644 --- a/backend/.sqlx/query-a00e61e770e20157bbd9e4cdedf7fb5f9de7c8c9e50282e3ecf2e3ce917ec37a.json +++ b/backend/.sqlx/query-24f38f0642b49626c8c8417e1846ab38dfe15284a6a7f54366ba25d7eb75a74a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = $2", + "query": "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "a00e61e770e20157bbd9e4cdedf7fb5f9de7c8c9e50282e3ecf2e3ce917ec37a" + "hash": "24f38f0642b49626c8c8417e1846ab38dfe15284a6a7f54366ba25d7eb75a74a" } diff --git a/backend/.sqlx/query-25b7c964336321fa10ea988831526b391cc1f02185ee87dbbda3d8a388cc858a.json b/backend/.sqlx/query-25b7c964336321fa10ea988831526b391cc1f02185ee87dbbda3d8a388cc858a.json new file mode 100644 index 0000000000..fb2ab7fcf5 --- /dev/null +++ b/backend/.sqlx/query-25b7c964336321fa10ea988831526b391cc1f02185ee87dbbda3d8a388cc858a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM v2_job_completed WHERE id = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "25b7c964336321fa10ea988831526b391cc1f02185ee87dbbda3d8a388cc858a" +} diff --git a/backend/.sqlx/query-af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49.json b/backend/.sqlx/query-25bf02e605e9e8e708a5dcfbb898b1af55045c1b8c2a138ca995882dde955971.json similarity index 64% rename from backend/.sqlx/query-af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49.json rename to backend/.sqlx/query-25bf02e605e9e8e708a5dcfbb898b1af55045c1b8c2a138ca995882dde955971.json index 2c3c76ca31..660044efb2 100644 --- a/backend/.sqlx/query-af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49.json +++ b/backend/.sqlx/query-25bf02e605e9e8e708a5dcfbb898b1af55045c1b8c2a138ca995882dde955971.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2", + "query": "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = EXCLUDED.config", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49" + "hash": "25bf02e605e9e8e708a5dcfbb898b1af55045c1b8c2a138ca995882dde955971" } diff --git a/backend/.sqlx/query-f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0.json b/backend/.sqlx/query-25cba74bec5959e6752265cd7b6f84846f74d468f0073f02f81122895e86c364.json similarity index 50% rename from backend/.sqlx/query-f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0.json rename to backend/.sqlx/query-25cba74bec5959e6752265cd7b6f84846f74d468f0073f02f81122895e86c364.json index 804f35777e..7c07d169b8 100644 --- a/backend/.sqlx/query-f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0.json +++ b/backend/.sqlx/query-25cba74bec5959e6752265cd7b6f84846f74d468f0073f02f81122895e86c364.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT flow_status->'user_states'->$1\n FROM v2_as_queue\n WHERE id = $2 AND workspace_id = $3\n ", + "query": "\n SELECT COALESCE(s.flow_status, s.workflow_as_code_status)->'user_states'->$1\n FROM v2_job_queue q LEFT JOIN v2_job_status s USING (id)\n WHERE q.id = $2 AND q.workspace_id = $3\n ", "describe": { "columns": [ { @@ -20,5 +20,5 @@ null ] }, - "hash": "f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0" + "hash": "25cba74bec5959e6752265cd7b6f84846f74d468f0073f02f81122895e86c364" } diff --git a/backend/.sqlx/query-2659fe2e121ac15da08030c9e72bdb79a580711cba2139f0cf901b30bb491fd5.json b/backend/.sqlx/query-2659fe2e121ac15da08030c9e72bdb79a580711cba2139f0cf901b30bb491fd5.json deleted file mode 100644 index f1d2942f51..0000000000 --- a/backend/.sqlx/query-2659fe2e121ac15da08030c9e72bdb79a580711cba2139f0cf901b30bb491fd5.json +++ /dev/null @@ -1,150 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n mqtt_resource_path,\n subscribe_topics as \"subscribe_topics!: Vec>\",\n v3_config as \"v3_config!: Option>\",\n v5_config as \"v5_config!: Option>\",\n client_version as \"client_version: _\",\n client_id,\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM\n mqtt_trigger\n WHERE\n enabled IS TRUE\n AND (last_server_ping IS NULL OR\n last_server_ping < now() - interval '15 seconds'\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "mqtt_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "subscribe_topics!: Vec>", - "type_info": "JsonbArray" - }, - { - "ordinal": 2, - "name": "v3_config!: Option>", - "type_info": "Jsonb" - }, - { - "ordinal": 3, - "name": "v5_config!: Option>", - "type_info": "Jsonb" - }, - { - "ordinal": 4, - "name": "client_version: _", - "type_info": { - "Custom": { - "name": "mqtt_client_version", - "kind": { - "Enum": [ - "v3", - "v5" - ] - } - } - } - }, - { - "ordinal": 5, - "name": "client_id", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 10, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 12, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 13, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 14, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 15, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 16, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 17, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 18, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 19, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 20, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - true, - true, - false, - true, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - true, - true, - true - ] - }, - "hash": "2659fe2e121ac15da08030c9e72bdb79a580711cba2139f0cf901b30bb491fd5" -} diff --git a/backend/.sqlx/query-2709e8113527d4cb331c72009e95e2efe4d1b57d5da6051acfb23f89b66434fb.json b/backend/.sqlx/query-2709e8113527d4cb331c72009e95e2efe4d1b57d5da6051acfb23f89b66434fb.json new file mode 100644 index 0000000000..0d2140ee9a --- /dev/null +++ b/backend/.sqlx/query-2709e8113527d4cb331c72009e95e2efe4d1b57d5da6051acfb23f89b66434fb.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM instance_group WHERE name = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "2709e8113527d4cb331c72009e95e2efe4d1b57d5da6051acfb23f89b66434fb" +} diff --git a/backend/.sqlx/query-2729c73b53605908e2fa26b4e4bbbd75d6d5bb77e6a92bf187d978a059d7af4a.json b/backend/.sqlx/query-2729c73b53605908e2fa26b4e4bbbd75d6d5bb77e6a92bf187d978a059d7af4a.json new file mode 100644 index 0000000000..afa445d679 --- /dev/null +++ b/backend/.sqlx/query-2729c73b53605908e2fa26b4e4bbbd75d6d5bb77e6a92bf187d978a059d7af4a.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM v2_job_queue q WHERE q.canceled_by IS NULL AND (q.scheduled_for <= now()\n OR (q.suspend_until IS NOT NULL\n AND (q.suspend <= 0 OR q.suspend_until <= now())))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "2729c73b53605908e2fa26b4e4bbbd75d6d5bb77e6a92bf187d978a059d7af4a" +} diff --git a/backend/.sqlx/query-e64f7044c74e96c2338580562f6b087805dad2b6fb1aa194ac2a2026fa24ecd0.json b/backend/.sqlx/query-27a54f8188c25c2c089c818a991ca1c092f67227be217161d6e6617ddbf77b32.json similarity index 78% rename from backend/.sqlx/query-e64f7044c74e96c2338580562f6b087805dad2b6fb1aa194ac2a2026fa24ecd0.json rename to backend/.sqlx/query-27a54f8188c25c2c089c818a991ca1c092f67227be217161d6e6617ddbf77b32.json index 4caab26834..c540742fe2 100644 --- a/backend/.sqlx/query-e64f7044c74e96c2338580562f6b087805dad2b6fb1aa194ac2a2026fa24ecd0.json +++ b/backend/.sqlx/query-27a54f8188c25c2c089c818a991ca1c092f67227be217161d6e6617ddbf77b32.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, timeout, on_behalf_of_email, created_by FROM script\n WHERE path = $1 AND workspace_id = $2 AND archived = false AND (lock IS NOT NULL OR $3 = false)\n ORDER BY created_at DESC LIMIT 1", + "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, debounce_key, debounce_delay_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, timeout, on_behalf_of_email, created_by FROM script\n WHERE path = $1 AND workspace_id = $2 AND archived = false AND (lock IS NOT NULL OR $3 = false)\n ORDER BY created_at DESC LIMIT 1", "describe": { "columns": [ { @@ -30,11 +30,21 @@ }, { "ordinal": 5, + "name": "debounce_key", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "debounce_delay_s", + "type_info": "Int4" + }, + { + "ordinal": 7, "name": "cache_ttl", "type_info": "Int4" }, { - "ordinal": 6, + "ordinal": 8, "name": "language: ScriptLang", "type_info": { "Custom": { @@ -70,27 +80,27 @@ } }, { - "ordinal": 7, + "ordinal": 9, "name": "dedicated_worker", "type_info": "Bool" }, { - "ordinal": 8, + "ordinal": 10, "name": "priority", "type_info": "Int2" }, { - "ordinal": 9, + "ordinal": 11, "name": "timeout", "type_info": "Int4" }, { - "ordinal": 10, + "ordinal": 12, "name": "on_behalf_of_email", "type_info": "Text" }, { - "ordinal": 11, + "ordinal": 13, "name": "created_by", "type_info": "Varchar" } @@ -109,6 +119,8 @@ true, true, true, + true, + true, false, true, true, @@ -117,5 +129,5 @@ false ] }, - "hash": "e64f7044c74e96c2338580562f6b087805dad2b6fb1aa194ac2a2026fa24ecd0" + "hash": "27a54f8188c25c2c089c818a991ca1c092f67227be217161d6e6617ddbf77b32" } diff --git a/backend/.sqlx/query-27b0c827467cc92979f094620957bc0edfa295d6c2292e509a5536765d120bd8.json b/backend/.sqlx/query-27b0c827467cc92979f094620957bc0edfa295d6c2292e509a5536765d120bd8.json new file mode 100644 index 0000000000..6d127f4c83 --- /dev/null +++ b/backend/.sqlx/query-27b0c827467cc92979f094620957bc0edfa295d6c2292e509a5536765d120bd8.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT draft_only FROM app WHERE path = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "draft_only", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "27b0c827467cc92979f094620957bc0edfa295d6c2292e509a5536765d120bd8" +} diff --git a/backend/.sqlx/query-28e81c9e4a9d166f38fff7e7f1cf87437ac0e47f59eb9d15ad07856a690319ce.json b/backend/.sqlx/query-28e81c9e4a9d166f38fff7e7f1cf87437ac0e47f59eb9d15ad07856a690319ce.json deleted file mode 100644 index a139571c83..0000000000 --- a/backend/.sqlx/query-28e81c9e4a9d166f38fff7e7f1cf87437ac0e47f59eb9d15ad07856a690319ce.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT flow_path, runnable_path, script_hash, runnable_is_flow, app_path\n FROM workspace_runnable_dependencies \n WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "flow_path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "runnable_path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "script_hash", - "type_info": "Int8" - }, - { - "ordinal": 3, - "name": "runnable_is_flow", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "app_path", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - true, - false, - true, - false, - true - ] - }, - "hash": "28e81c9e4a9d166f38fff7e7f1cf87437ac0e47f59eb9d15ad07856a690319ce" -} diff --git a/backend/.sqlx/query-28f1ecca40c8b81cc59dffb75e2913c889b374999ece04173b2e67dc74005f60.json b/backend/.sqlx/query-28f1ecca40c8b81cc59dffb75e2913c889b374999ece04173b2e67dc74005f60.json new file mode 100644 index 0000000000..a5c5e9427e --- /dev/null +++ b/backend/.sqlx/query-28f1ecca40c8b81cc59dffb75e2913c889b374999ece04173b2e67dc74005f60.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT draft_only FROM flow WHERE path = $1 AND workspace_id = $2 AND archived = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "draft_only", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "28f1ecca40c8b81cc59dffb75e2913c889b374999ece04173b2e67dc74005f60" +} diff --git a/backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json b/backend/.sqlx/query-29785f22ee7092e8cebeae3757caf0563e3f98fff1f100616c33e8dc95fbff99.json similarity index 58% rename from backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json rename to backend/.sqlx/query-29785f22ee7092e8cebeae3757caf0563e3f98fff1f100616c33e8dc95fbff99.json index 08bb98cc06..11f612ccda 100644 --- a/backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json +++ b/backend/.sqlx/query-29785f22ee7092e8cebeae3757caf0563e3f98fff1f100616c33e8dc95fbff99.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\"\n FROM v2_as_completed_job\n WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "query": "SELECT j.created_by AS \"created_by!\", j.args as \"args: sqlx::types::Json>\"\n FROM v2_job j\n WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", "describe": { "columns": [ { @@ -22,9 +22,9 @@ ] }, "nullable": [ - true, + false, true ] }, - "hash": "2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9" + "hash": "29785f22ee7092e8cebeae3757caf0563e3f98fff1f100616c33e8dc95fbff99" } diff --git a/backend/.sqlx/query-298f8609319a2928257fd5be60bb37f292c786d2348efe11d19868e5dc8fba11.json b/backend/.sqlx/query-298f8609319a2928257fd5be60bb37f292c786d2348efe11d19868e5dc8fba11.json deleted file mode 100644 index 605025b997..0000000000 --- a/backend/.sqlx/query-298f8609319a2928257fd5be60bb37f292c786d2348efe11d19868e5dc8fba11.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT elem FROM (SELECT unnest($1::TEXT[]) AS elem) AS e\n WHERE elem NOT IN (SELECT datname FROM pg_catalog.pg_database);", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "elem", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "TextArray" - ] - }, - "nullable": [ - null - ] - }, - "hash": "298f8609319a2928257fd5be60bb37f292c786d2348efe11d19868e5dc8fba11" -} diff --git a/backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json b/backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json deleted file mode 100644 index d64d8f404b..0000000000 --- a/backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'gcp' AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63" -} diff --git a/backend/.sqlx/query-29f096ec62c4abb1435a5667e2b30e9c1724e419cdc23ef1b300e84c02a20427.json b/backend/.sqlx/query-29f096ec62c4abb1435a5667e2b30e9c1724e419cdc23ef1b300e84c02a20427.json deleted file mode 100644 index b7075f9df1..0000000000 --- a/backend/.sqlx/query-29f096ec62c4abb1435a5667e2b30e9c1724e419cdc23ef1b300e84c02a20427.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'postgres'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "29f096ec62c4abb1435a5667e2b30e9c1724e419cdc23ef1b300e84c02a20427" -} diff --git a/backend/.sqlx/query-55a2f170823f1d1abce76287d8817a6cf34de92b9b4079c00b75423a9ff835b9.json b/backend/.sqlx/query-2b34ae324f90924dba6c4562024f72e7b051adb368b3685181047f1f4522a473.json similarity index 76% rename from backend/.sqlx/query-55a2f170823f1d1abce76287d8817a6cf34de92b9b4079c00b75423a9ff835b9.json rename to backend/.sqlx/query-2b34ae324f90924dba6c4562024f72e7b051adb368b3685181047f1f4522a473.json index 7b7dd7fe29..302c0b461d 100644 --- a/backend/.sqlx/query-55a2f170823f1d1abce76287d8817a6cf34de92b9b4079c00b75423a9ff835b9.json +++ b/backend/.sqlx/query-2b34ae324f90924dba6c4562024f72e7b051adb368b3685181047f1f4522a473.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO variable\n (workspace_id, path, value, is_secret, description, account, is_oauth)\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n ON CONFLICT (workspace_id, path) DO UPDATE SET value = $3", + "query": "INSERT INTO variable\n (workspace_id, path, value, is_secret, description, account, is_oauth)\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n ON CONFLICT (workspace_id, path) DO UPDATE SET value = EXCLUDED.value", "describe": { "columns": [], "parameters": { @@ -16,5 +16,5 @@ }, "nullable": [] }, - "hash": "55a2f170823f1d1abce76287d8817a6cf34de92b9b4079c00b75423a9ff835b9" + "hash": "2b34ae324f90924dba6c4562024f72e7b051adb368b3685181047f1f4522a473" } diff --git a/backend/.sqlx/query-8d119104337bf99e9aa9dcbac0a54154267a7db96cc0fb3ebaac95635e24da29.json b/backend/.sqlx/query-2e131a019051bdac7c9c65f7c504cfba31cfdd64a0f68001e123694ed5cde5ed.json similarity index 75% rename from backend/.sqlx/query-8d119104337bf99e9aa9dcbac0a54154267a7db96cc0fb3ebaac95635e24da29.json rename to backend/.sqlx/query-2e131a019051bdac7c9c65f7c504cfba31cfdd64a0f68001e123694ed5cde5ed.json index 1c2f9a9f33..347ad02c31 100644 --- a/backend/.sqlx/query-8d119104337bf99e9aa9dcbac0a54154267a7db96cc0fb3ebaac95635e24da29.json +++ b/backend/.sqlx/query-2e131a019051bdac7c9c65f7c504cfba31cfdd64a0f68001e123694ed5cde5ed.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5) \n ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL DO UPDATE SET callback_job_ids = $4, deployment_msg = $5", + "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5) \n ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL DO UPDATE SET callback_job_ids = EXCLUDED.callback_job_ids, deployment_msg = EXCLUDED.deployment_msg", "describe": { "columns": [], "parameters": { @@ -14,5 +14,5 @@ }, "nullable": [] }, - "hash": "8d119104337bf99e9aa9dcbac0a54154267a7db96cc0fb3ebaac95635e24da29" + "hash": "2e131a019051bdac7c9c65f7c504cfba31cfdd64a0f68001e123694ed5cde5ed" } diff --git a/backend/.sqlx/query-2e589e039986e7a2c75e328868874669b32cbe0dae6822b2d2fad0635c5f6087.json b/backend/.sqlx/query-2e589e039986e7a2c75e328868874669b32cbe0dae6822b2d2fad0635c5f6087.json new file mode 100644 index 0000000000..6ba559e644 --- /dev/null +++ b/backend/.sqlx/query-2e589e039986e7a2c75e328868874669b32cbe0dae6822b2d2fad0635c5f6087.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE v2_job_status\n SET flow_status = jsonb_set(flow_status, array['stream_job'], to_jsonb($1::UUID::TEXT))\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "2e589e039986e7a2c75e328868874669b32cbe0dae6822b2d2fad0635c5f6087" +} diff --git a/backend/.sqlx/query-f3f96e066716e61042519a645d487b578bc63792cdb0f7ddaeb82e9771287c22.json b/backend/.sqlx/query-3162ec92bb32af47a71cc41172cc740b5dea1304ce4dfdb4d3d0efa4266f38c5.json similarity index 93% rename from backend/.sqlx/query-f3f96e066716e61042519a645d487b578bc63792cdb0f7ddaeb82e9771287c22.json rename to backend/.sqlx/query-3162ec92bb32af47a71cc41172cc740b5dea1304ce4dfdb4d3d0efa4266f38c5.json index 28c0756b0f..fa013ba585 100644 --- a/backend/.sqlx/query-f3f96e066716e61042519a645d487b578bc63792cdb0f7ddaeb82e9771287c22.json +++ b/backend/.sqlx/query-3162ec92bb32af47a71cc41172cc740b5dea1304ce4dfdb4d3d0efa4266f38c5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT \n v2_job_queue.workspace_id,\n v2_job_queue.id,\n v2_job.args as \"args: sqlx::types::Json>>\",\n v2_job.parent_job,\n v2_job.created_by,\n v2_job_queue.started_at,\n scheduled_for,\n runnable_path,\n kind as \"kind: JobKind\",\n runnable_id as \"runnable_id: ScriptHash\",\n canceled_reason,\n canceled_by,\n permissioned_as,\n permissioned_as_email,\n flow_status as \"flow_status: sqlx::types::Json>\",\n v2_job.tag,\n script_lang as \"script_lang: ScriptLang\",\n same_worker,\n pre_run_error,\n concurrent_limit,\n concurrency_time_window_s,\n flow_innermost_root_job,\n root_job,\n timeout,\n flow_step_id,\n cache_ttl,\n v2_job_queue.priority,\n preprocessed,\n script_entrypoint_override,\n trigger,\n trigger_kind as \"trigger_kind: JobTriggerKind\",\n visible_to_owner\n FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id WHERE v2_job_queue.id = $1", + "query": "SELECT \n v2_job_queue.workspace_id,\n v2_job_queue.id,\n v2_job.args as \"args: sqlx::types::Json>>\",\n v2_job.parent_job,\n v2_job.created_by,\n v2_job_queue.started_at,\n scheduled_for,\n runnable_path,\n kind as \"kind: JobKind\",\n runnable_id as \"runnable_id: ScriptHash\",\n canceled_reason,\n canceled_by,\n permissioned_as,\n permissioned_as_email,\n flow_status as \"flow_status: sqlx::types::Json>\",\n v2_job.tag,\n script_lang as \"script_lang: ScriptLang\",\n same_worker,\n pre_run_error,\n concurrent_limit,\n concurrency_time_window_s,\n flow_innermost_root_job,\n root_job,\n timeout,\n flow_step_id,\n cache_ttl,\n v2_job_queue.priority,\n preprocessed,\n script_entrypoint_override,\n trigger,\n trigger_kind as \"trigger_kind: JobTriggerKind\",\n visible_to_owner,\n NULL as permissioned_as_end_user_email\n FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id WHERE v2_job_queue.id = $1", "describe": { "columns": [ { @@ -65,7 +65,7 @@ "noop", "appdependencies", "deploymentcallback", - "singlescriptflow", + "singlestepflow", "flowscript", "flownode", "appscript", @@ -241,6 +241,11 @@ "ordinal": 31, "name": "visible_to_owner", "type_info": "Bool" + }, + { + "ordinal": 32, + "name": "permissioned_as_end_user_email", + "type_info": "Text" } ], "parameters": { @@ -280,8 +285,9 @@ true, true, true, - false + false, + null ] }, - "hash": "f3f96e066716e61042519a645d487b578bc63792cdb0f7ddaeb82e9771287c22" + "hash": "3162ec92bb32af47a71cc41172cc740b5dea1304ce4dfdb4d3d0efa4266f38c5" } diff --git a/backend/.sqlx/query-31869c5dba5cefd4ffaae7720617a40ef42ca940ca4ff7f9fb2e69e63d830d27.json b/backend/.sqlx/query-31869c5dba5cefd4ffaae7720617a40ef42ca940ca4ff7f9fb2e69e63d830d27.json deleted file mode 100644 index e1bf471dd5..0000000000 --- a/backend/.sqlx/query-31869c5dba5cefd4ffaae7720617a40ef42ca940ca4ff7f9fb2e69e63d830d27.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at\n FROM variable \n WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "value", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "is_secret", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "description", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 6, - "name": "account", - "type_info": "Int4" - }, - { - "ordinal": 7, - "name": "is_oauth", - "type_info": "Bool" - }, - { - "ordinal": 8, - "name": "expires_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - true, - false, - true - ] - }, - "hash": "31869c5dba5cefd4ffaae7720617a40ef42ca940ca4ff7f9fb2e69e63d830d27" -} diff --git a/backend/.sqlx/query-326fd614ebd965b9bb6f3e578f75a54d80812ff144e711100e6ac659785c991d.json b/backend/.sqlx/query-326fd614ebd965b9bb6f3e578f75a54d80812ff144e711100e6ac659785c991d.json new file mode 100644 index 0000000000..b7645b6a44 --- /dev/null +++ b/backend/.sqlx/query-326fd614ebd965b9bb6f3e578f75a54d80812ff144e711100e6ac659785c991d.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE usr SET added_via = $1, is_admin = $2, operator = $3 WHERE username = $4 AND workspace_id = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Bool", + "Bool", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "326fd614ebd965b9bb6f3e578f75a54d80812ff144e711100e6ac659785c991d" +} diff --git a/backend/.sqlx/query-33aaf2dd14397d0b50b986ed55bc458337b85b6a9a9ab3d93d8f33d0f57e4b0f.json b/backend/.sqlx/query-33aaf2dd14397d0b50b986ed55bc458337b85b6a9a9ab3d93d8f33d0f57e4b0f.json deleted file mode 100644 index 188dcc8d5b..0000000000 --- a/backend/.sqlx/query-33aaf2dd14397d0b50b986ed55bc458337b85b6a9a9ab3d93d8f33d0f57e4b0f.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n gcp_resource_path,\n subscription_id,\n topic_id,\n workspace_id,\n delivery_type AS \"delivery_type: _\",\n delivery_config AS \"delivery_config: _\",\n subscription_mode AS \"subscription_mode: _\",\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\",\n auto_acknowledge_msg\n FROM\n gcp_trigger\n WHERE\n delivery_type != 'push'::DELIVERY_MODE AND\n enabled IS TRUE\n AND (last_server_ping IS NULL OR\n last_server_ping < now() - interval '15 seconds'\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "gcp_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "subscription_id", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "topic_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "delivery_type: _", - "type_info": { - "Custom": { - "name": "delivery_mode", - "kind": { - "Enum": [ - "push", - "pull" - ] - } - } - } - }, - { - "ordinal": 5, - "name": "delivery_config: _", - "type_info": "Jsonb" - }, - { - "ordinal": 6, - "name": "subscription_mode: _", - "type_info": { - "Custom": { - "name": "gcp_subscription_mode", - "kind": { - "Enum": [ - "create_update", - "existing" - ] - } - } - } - }, - { - "ordinal": 7, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 10, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 12, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 13, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 14, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 15, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 16, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 17, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 18, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 19, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 20, - "name": "retry: _", - "type_info": "Jsonb" - }, - { - "ordinal": 21, - "name": "auto_acknowledge_msg", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - false, - true, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - true, - true, - true, - true - ] - }, - "hash": "33aaf2dd14397d0b50b986ed55bc458337b85b6a9a9ab3d93d8f33d0f57e4b0f" -} diff --git a/backend/.sqlx/query-34f4c05b844a83627a5c61edbfe070d0a52e20deaf550185206ae854cce274be.json b/backend/.sqlx/query-34f4c05b844a83627a5c61edbfe070d0a52e20deaf550185206ae854cce274be.json new file mode 100644 index 0000000000..b5d94b1915 --- /dev/null +++ b/backend/.sqlx/query-34f4c05b844a83627a5c61edbfe070d0a52e20deaf550185206ae854cce274be.json @@ -0,0 +1,249 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n j.parent_job,\n j.created_by,\n cj.duration_ms,\n cj.status = 'success' OR cj.status = 'skipped' AS success,\n j.runnable_id AS script_hash,\n j.runnable_path AS script_path,\n j.args,\n cj.completed_at,\n cj.result,\n cj.deleted,\n cj.status = 'canceled' AS canceled,\n cj.canceled_by,\n cj.canceled_reason,\n j.kind AS job_kind,\n CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END AS schedule_path,\n j.permissioned_as,\n j.flow_step_id IS NOT NULL AS is_flow_step,\n j.script_lang AS language,\n cj.status = 'skipped' AS is_skipped,\n j.permissioned_as_email AS email,\n j.visible_to_owner,\n cj.memory_peak AS mem_peak,\n j.tag,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(j.created_by), 0) + \n COALESCE(LENGTH(j.runnable_path), 0) + \n COALESCE(LENGTH(j.args::text), 0) + \n COALESCE(LENGTH(cj.result::text), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_job_completed AS cj\n JOIN v2_job AS j ON cj.id = j.id\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE cj.completed_at < $1\n ORDER BY completed_at ASC\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT \n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY completed_at ASC \n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes,\n ROW_NUMBER() OVER (ORDER BY completed_at ASC) AS row_num\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option\",\n script_path,\n args AS \"args: sqlx::types::Json>>\",\n result AS \"result: sqlx::types::Json>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n completed_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $2 OR row_num = 1\n ORDER BY completed_at ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "success!", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "script_hash!: Option", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "args: sqlx::types::Json>>", + "type_info": "Jsonb" + }, + { + "ordinal": 9, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 10, + "name": "deleted!", + "type_info": "Bool" + }, + { + "ordinal": 11, + "name": "canceled!", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "canceled_reason", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "job_kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent" + ] + } + } + } + }, + { + "ordinal": 15, + "name": "schedule_path", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "permissioned_as!", + "type_info": "Varchar" + }, + { + "ordinal": 17, + "name": "is_flow_step!", + "type_info": "Bool" + }, + { + "ordinal": 18, + "name": "language: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + }, + { + "ordinal": 19, + "name": "is_skipped!", + "type_info": "Bool" + }, + { + "ordinal": 20, + "name": "email!", + "type_info": "Varchar" + }, + { + "ordinal": 21, + "name": "visible_to_owner!", + "type_info": "Bool" + }, + { + "ordinal": 22, + "name": "mem_peak", + "type_info": "Int4" + }, + { + "ordinal": 23, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 24, + "name": "created_at!", + "type_info": "Timestamptz" + }, + { + "ordinal": 25, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 26, + "name": "logs", + "type_info": "Text" + }, + { + "ordinal": 27, + "name": "log_offset?", + "type_info": "Int4" + }, + { + "ordinal": 28, + "name": "log_file_index", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Timestamptz", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + false, + null, + true, + true, + true, + true, + false, + null, + true, + true, + false, + null, + false, + null, + true, + null, + false, + false, + true, + false, + false, + true, + true, + false, + true + ] + }, + "hash": "34f4c05b844a83627a5c61edbfe070d0a52e20deaf550185206ae854cce274be" +} diff --git a/backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json b/backend/.sqlx/query-35061719d01929a7146c80de4b637abdad3198d3340ec7c04ed671baff0a4d0b.json similarity index 54% rename from backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json rename to backend/.sqlx/query-35061719d01929a7146c80de4b637abdad3198d3340ec7c04ed671baff0a4d0b.json index a76515fd7e..e3d94ad1cf 100644 --- a/backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json +++ b/backend/.sqlx/query-35061719d01929a7146c80de4b637abdad3198d3340ec7c04ed671baff0a4d0b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM v2_as_completed_job\n LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id\n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))", + "query": "SELECT j.created_by AS \"created_by!\", CONCAT(coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM v2_job j\n LEFT JOIN job_logs ON job_logs.job_id = j.id\n WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", "describe": { "columns": [ { @@ -32,11 +32,11 @@ ] }, "nullable": [ - true, + false, null, false, true ] }, - "hash": "ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9" + "hash": "35061719d01929a7146c80de4b637abdad3198d3340ec7c04ed671baff0a4d0b" } diff --git a/backend/.sqlx/query-280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866.json b/backend/.sqlx/query-371d652fbb1d34d56f645c75d12852ff982821ed7b24574cdd09a06eac0d628b.json similarity index 64% rename from backend/.sqlx/query-280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866.json rename to backend/.sqlx/query-371d652fbb1d34d56f645c75d12852ff982821ed7b24574cdd09a06eac0d628b.json index e9705c23a6..30550d5011 100644 --- a/backend/.sqlx/query-280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866.json +++ b/backend/.sqlx/query-371d652fbb1d34d56f645c75d12852ff982821ed7b24574cdd09a06eac0d628b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT script_path FROM v2_as_completed_job WHERE id = $1", + "query": "SELECT runnable_path as script_path FROM v2_job WHERE id = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ true ] }, - "hash": "280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866" + "hash": "371d652fbb1d34d56f645c75d12852ff982821ed7b24574cdd09a06eac0d628b" } diff --git a/backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json b/backend/.sqlx/query-37285436c16684449b33810d97d0a2611dab30faff2891fc3a7f00ee8c120950.json similarity index 55% rename from backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json rename to backend/.sqlx/query-37285436c16684449b33810d97d0a2611dab30faff2891fc3a7f00ee8c120950.json index ee5e15118d..c26f747c98 100644 --- a/backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json +++ b/backend/.sqlx/query-37285436c16684449b33810d97d0a2611dab30faff2891fc3a7f00ee8c120950.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM v2_as_queue\n LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id\n WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))", + "query": "SELECT j.created_by AS \"created_by!\", CONCAT(coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM v2_job j\n LEFT JOIN job_logs ON job_logs.job_id = j.id\n WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", "describe": { "columns": [ { @@ -32,11 +32,11 @@ ] }, "nullable": [ - true, + false, null, null, true ] }, - "hash": "2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f" + "hash": "37285436c16684449b33810d97d0a2611dab30faff2891fc3a7f00ee8c120950" } diff --git a/backend/.sqlx/query-df5b933f81ca7e3bbb3fb522baedf749fa3bbbf2c0e43d5d4ea148b5bc990067.json b/backend/.sqlx/query-37e23397905e25bbbf5a7047c790967a97cc8f6948beef706b0c053621882330.json similarity index 79% rename from backend/.sqlx/query-df5b933f81ca7e3bbb3fb522baedf749fa3bbbf2c0e43d5d4ea148b5bc990067.json rename to backend/.sqlx/query-37e23397905e25bbbf5a7047c790967a97cc8f6948beef706b0c053621882330.json index 7918b97267..9474727ace 100644 --- a/backend/.sqlx/query-df5b933f81ca7e3bbb3fb522baedf749fa3bbbf2c0e43d5d4ea148b5bc990067.json +++ b/backend/.sqlx/query-37e23397905e25bbbf5a7047c790967a97cc8f6948beef706b0c053621882330.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only FROM password WHERE email = $1", + "query": "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user FROM password WHERE email = $1", "describe": { "columns": [ { @@ -47,6 +47,11 @@ "ordinal": 8, "name": "operator_only", "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "first_time_user", + "type_info": "Bool" } ], "parameters": { @@ -63,8 +68,9 @@ true, true, true, - null + null, + false ] }, - "hash": "df5b933f81ca7e3bbb3fb522baedf749fa3bbbf2c0e43d5d4ea148b5bc990067" + "hash": "37e23397905e25bbbf5a7047c790967a97cc8f6948beef706b0c053621882330" } diff --git a/backend/.sqlx/query-383b8adaadc6e23952ca8942fe7dbaa7c74ce0cfa1b945b7514e19a70f7a6f1c.json b/backend/.sqlx/query-383b8adaadc6e23952ca8942fe7dbaa7c74ce0cfa1b945b7514e19a70f7a6f1c.json deleted file mode 100644 index db96fe6009..0000000000 --- a/backend/.sqlx/query-383b8adaadc6e23952ca8942fe7dbaa7c74ce0cfa1b945b7514e19a70f7a6f1c.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO instance_group (name, summary) VALUES ($1, $2) ON CONFLICT DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "383b8adaadc6e23952ca8942fe7dbaa7c74ce0cfa1b945b7514e19a70f7a6f1c" -} diff --git a/backend/.sqlx/query-652835b2b7f801532a591988ac76d385188991c6654d529f6d65f6f03794844a.json b/backend/.sqlx/query-384753e7ceb602790646b0f269df48910d900b9575d986143823781e1d005d57.json similarity index 74% rename from backend/.sqlx/query-652835b2b7f801532a591988ac76d385188991c6654d529f6d65f6f03794844a.json rename to backend/.sqlx/query-384753e7ceb602790646b0f269df48910d900b9575d986143823781e1d005d57.json index 91e268889d..d90cb38e55 100644 --- a/backend/.sqlx/query-652835b2b7f801532a591988ac76d385188991c6654d529f6d65f6f03794844a.json +++ b/backend/.sqlx/query-384753e7ceb602790646b0f269df48910d900b9575d986143823781e1d005d57.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO deployment_metadata (workspace_id, path, app_version, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path, app_version) WHERE app_version IS NOT NULL DO UPDATE SET deployment_msg = $4", + "query": "INSERT INTO deployment_metadata (workspace_id, path, app_version, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path, app_version) WHERE app_version IS NOT NULL DO UPDATE SET deployment_msg = EXCLUDED.deployment_msg", "describe": { "columns": [], "parameters": { @@ -13,5 +13,5 @@ }, "nullable": [] }, - "hash": "652835b2b7f801532a591988ac76d385188991c6654d529f6d65f6f03794844a" + "hash": "384753e7ceb602790646b0f269df48910d900b9575d986143823781e1d005d57" } diff --git a/backend/.sqlx/query-42df4b40b3bbf14010f07e29892776992bfeb383d590e379244ad23703e536a4.json b/backend/.sqlx/query-3904d59575e05df6e414be03e84ca7d07ee06d417a5bd9f02d64fcb467cf362a.json similarity index 73% rename from backend/.sqlx/query-42df4b40b3bbf14010f07e29892776992bfeb383d590e379244ad23703e536a4.json rename to backend/.sqlx/query-3904d59575e05df6e414be03e84ca7d07ee06d417a5bd9f02d64fcb467cf362a.json index 695b4d2969..f6284b98a7 100644 --- a/backend/.sqlx/query-42df4b40b3bbf14010f07e29892776992bfeb383d590e379244ad23703e536a4.json +++ b/backend/.sqlx/query-3904d59575e05df6e414be03e84ca7d07ee06d417a5bd9f02d64fcb467cf362a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at)\n VALUES ($1, $2, $3, $4, $5, $6, now()) ON CONFLICT (workspace_id, path) DO UPDATE SET value = $3, edited_at = now()", + "query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at)\n VALUES ($1, $2, $3, $4, $5, $6, now()) ON CONFLICT (workspace_id, path) DO UPDATE SET value = EXCLUDED.value, edited_at = now()", "describe": { "columns": [], "parameters": { @@ -15,5 +15,5 @@ }, "nullable": [] }, - "hash": "42df4b40b3bbf14010f07e29892776992bfeb383d590e379244ad23703e536a4" + "hash": "3904d59575e05df6e414be03e84ca7d07ee06d417a5bd9f02d64fcb467cf362a" } diff --git a/backend/.sqlx/query-c69719d0a63b0ca434c3317529e00e4d0df0104b6c1dbdf6d0f68f5047a2ad5e.json b/backend/.sqlx/query-39426bd3018b390ea2073419884cf6cb506c75e84c65438c9026831eb10d340b.json similarity index 64% rename from backend/.sqlx/query-c69719d0a63b0ca434c3317529e00e4d0df0104b6c1dbdf6d0f68f5047a2ad5e.json rename to backend/.sqlx/query-39426bd3018b390ea2073419884cf6cb506c75e84c65438c9026831eb10d340b.json index d3634c6e96..69064f7d45 100644 --- a/backend/.sqlx/query-c69719d0a63b0ca434c3317529e00e4d0df0104b6c1dbdf6d0f68f5047a2ad5e.json +++ b/backend/.sqlx/query-39426bd3018b390ea2073419884cf6cb506c75e84c65438c9026831eb10d340b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at)\n VALUES ($1, $2, $3, $4, $5, $6, now()) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = $3, description = $4, resource_type = $5, edited_at = now()", + "query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at)\n VALUES ($1, $2, $3, $4, $5, $6, now()) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now()", "describe": { "columns": [], "parameters": { @@ -15,5 +15,5 @@ }, "nullable": [] }, - "hash": "c69719d0a63b0ca434c3317529e00e4d0df0104b6c1dbdf6d0f68f5047a2ad5e" + "hash": "39426bd3018b390ea2073419884cf6cb506c75e84c65438c9026831eb10d340b" } diff --git a/backend/.sqlx/query-3a5edf3dd884b5a8862bb112f6520967ed4a218782192c6c6fc1498f45d753a6.json b/backend/.sqlx/query-3a5edf3dd884b5a8862bb112f6520967ed4a218782192c6c6fc1498f45d753a6.json deleted file mode 100644 index 26d0b63b5c..0000000000 --- a/backend/.sqlx/query-3a5edf3dd884b5a8862bb112f6520967ed4a218782192c6c6fc1498f45d753a6.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "create index concurrently if not exists ix_job_created_at ON v2_job (created_at DESC)", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "3a5edf3dd884b5a8862bb112f6520967ed4a218782192c6c6fc1498f45d753a6" -} diff --git a/backend/.sqlx/query-3c5b6001aac7fb58ec9bfad1bfd7418c16f96f307bafa926a18316055b7c90c4.json b/backend/.sqlx/query-3c5b6001aac7fb58ec9bfad1bfd7418c16f96f307bafa926a18316055b7c90c4.json new file mode 100644 index 0000000000..c6f9924347 --- /dev/null +++ b/backend/.sqlx/query-3c5b6001aac7fb58ec9bfad1bfd7418c16f96f307bafa926a18316055b7c90c4.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dependency_map WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "3c5b6001aac7fb58ec9bfad1bfd7418c16f96f307bafa926a18316055b7c90c4" +} diff --git a/backend/.sqlx/query-3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c.json b/backend/.sqlx/query-3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c.json new file mode 100644 index 0000000000..a97d16b89c --- /dev/null +++ b/backend/.sqlx/query-3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO app_version\n (app_id, value, created_by, raw_app)\n SELECT app_id, value, created_by, raw_app\n FROM app_version WHERE id = $1\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c" +} diff --git a/backend/.sqlx/query-3dca0aded0ec744b084359e1a77dc4af312fda13d832f9e8b236655628e5b81c.json b/backend/.sqlx/query-3dca0aded0ec744b084359e1a77dc4af312fda13d832f9e8b236655628e5b81c.json new file mode 100644 index 0000000000..2488154bbd --- /dev/null +++ b/backend/.sqlx/query-3dca0aded0ec744b084359e1a77dc4af312fda13d832f9e8b236655628e5b81c.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_started_at_new_2", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "3dca0aded0ec744b084359e1a77dc4af312fda13d832f9e8b236655628e5b81c" +} diff --git a/backend/.sqlx/query-3e33469f448fa86c7e0d0deb65ee2c13b5dcc4cbc4fcb0fe2bde1fdcb6d20e74.json b/backend/.sqlx/query-3e33469f448fa86c7e0d0deb65ee2c13b5dcc4cbc4fcb0fe2bde1fdcb6d20e74.json deleted file mode 100644 index 113f888545..0000000000 --- a/backend/.sqlx/query-3e33469f448fa86c7e0d0deb65ee2c13b5dcc4cbc4fcb0fe2bde1fdcb6d20e74.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE \n capture_config \n SET \n last_server_ping = NULL \n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = 'mqtt' AND \n server_id IS NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "3e33469f448fa86c7e0d0deb65ee2c13b5dcc4cbc4fcb0fe2bde1fdcb6d20e74" -} diff --git a/backend/.sqlx/query-3e64c894c89ef82c4527180b82c4e82c1b7060ba0ca288dc6a7c7afcd76212e8.json b/backend/.sqlx/query-3e64c894c89ef82c4527180b82c4e82c1b7060ba0ca288dc6a7c7afcd76212e8.json deleted file mode 100644 index 8831155670..0000000000 --- a/backend/.sqlx/query-3e64c894c89ef82c4527180b82c4e82c1b7060ba0ca288dc6a7c7afcd76212e8.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE kafka_trigger SET enabled = FALSE, error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "3e64c894c89ef82c4527180b82c4e82c1b7060ba0ca288dc6a7c7afcd76212e8" -} diff --git a/backend/.sqlx/query-3f67e7cf8d9f021a075f1c88703287a9f42e252a1246a584249a224afdbbbf8a.json b/backend/.sqlx/query-3f67e7cf8d9f021a075f1c88703287a9f42e252a1246a584249a224afdbbbf8a.json deleted file mode 100644 index 8dec90897c..0000000000 --- a/backend/.sqlx/query-3f67e7cf8d9f021a075f1c88703287a9f42e252a1246a584249a224afdbbbf8a.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n sqs_trigger \n SET \n enabled = FALSE, \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "3f67e7cf8d9f021a075f1c88703287a9f42e252a1246a584249a224afdbbbf8a" -} diff --git a/backend/.sqlx/query-348f73fa8222ec195bb5b2260d1595e6e9fa9d73779cf352e94039fd47de4294.json b/backend/.sqlx/query-4144c87c25a939aafb2f57da189d94d038bcad7a36fbf87e0403c89a979c5b3f.json similarity index 78% rename from backend/.sqlx/query-348f73fa8222ec195bb5b2260d1595e6e9fa9d73779cf352e94039fd47de4294.json rename to backend/.sqlx/query-4144c87c25a939aafb2f57da189d94d038bcad7a36fbf87e0403c89a979c5b3f.json index 4f9e060ead..5ea52ccd39 100644 --- a/backend/.sqlx/query-348f73fa8222ec195bb5b2260d1595e6e9fa9d73779cf352e94039fd47de4294.json +++ b/backend/.sqlx/query-4144c87c25a939aafb2f57da189d94d038bcad7a36fbf87e0403c89a979c5b3f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE schedule SET\n schedule = $1,\n timezone = $2,\n args = $3,\n on_failure = $4,\n on_failure_times = $5,\n on_failure_exact = $6,\n on_failure_extra_args = $7,\n on_recovery = $8,\n on_recovery_times = $9,\n on_recovery_extra_args = $10,\n on_success = $11,\n on_success_extra_args = $12,\n ws_error_handler_muted = $13,\n retry = $14,\n summary = $15,\n no_flow_overlap = $16,\n tag = $17,\n paused_until = $18,\n path = $19,\n workspace_id = $20,\n cron_version = COALESCE($21, cron_version),\n description = $22\n WHERE path = $19 AND workspace_id = $20\n RETURNING\n workspace_id,\n path,\n edited_by,\n edited_at,\n schedule,\n timezone,\n enabled,\n script_path,\n is_flow,\n args AS \"args: _\",\n extra_perms,\n email,\n error,\n on_failure,\n on_failure_times,\n on_failure_exact,\n on_failure_extra_args AS \"on_failure_extra_args: _\",\n on_recovery,\n on_recovery_times,\n on_recovery_extra_args AS \"on_recovery_extra_args: _\",\n on_success,\n on_success_extra_args AS \"on_success_extra_args: _\",\n ws_error_handler_muted,\n retry,\n no_flow_overlap,\n summary,\n description,\n tag,\n paused_until,\n cron_version\n ", + "query": "\n UPDATE schedule SET\n schedule = $1,\n timezone = $2,\n args = $3,\n on_failure = $4,\n on_failure_times = $5,\n on_failure_exact = $6,\n on_failure_extra_args = $7,\n on_recovery = $8,\n on_recovery_times = $9,\n on_recovery_extra_args = $10,\n on_success = $11,\n on_success_extra_args = $12,\n ws_error_handler_muted = $13,\n retry = $14,\n summary = $15,\n no_flow_overlap = $16,\n tag = $17,\n paused_until = $18,\n path = $19,\n workspace_id = $20,\n cron_version = COALESCE($21, cron_version),\n description = $22,\n dynamic_skip = $23\n WHERE path = $19 AND workspace_id = $20\n RETURNING\n workspace_id,\n path,\n edited_by,\n edited_at,\n schedule,\n timezone,\n enabled,\n script_path,\n is_flow,\n args AS \"args: _\",\n extra_perms,\n email,\n error,\n on_failure,\n on_failure_times,\n on_failure_exact,\n on_failure_extra_args AS \"on_failure_extra_args: _\",\n on_recovery,\n on_recovery_times,\n on_recovery_extra_args AS \"on_recovery_extra_args: _\",\n on_success,\n on_success_extra_args AS \"on_success_extra_args: _\",\n ws_error_handler_muted,\n retry,\n no_flow_overlap,\n summary,\n description,\n tag,\n paused_until,\n cron_version,\n dynamic_skip\n ", "describe": { "columns": [ { @@ -152,6 +152,11 @@ "ordinal": 29, "name": "cron_version", "type_info": "Text" + }, + { + "ordinal": 30, + "name": "dynamic_skip", + "type_info": "Varchar" } ], "parameters": { @@ -177,7 +182,8 @@ "Text", "Text", "Text", - "Text" + "Text", + "Varchar" ] }, "nullable": [ @@ -210,8 +216,9 @@ true, true, true, + true, true ] }, - "hash": "348f73fa8222ec195bb5b2260d1595e6e9fa9d73779cf352e94039fd47de4294" + "hash": "4144c87c25a939aafb2f57da189d94d038bcad7a36fbf87e0403c89a979c5b3f" } diff --git a/backend/.sqlx/query-42030372693d8d6b8d03947bb6702024cfa226735eeeb720bbab46bd0e3cedcc.json b/backend/.sqlx/query-42030372693d8d6b8d03947bb6702024cfa226735eeeb720bbab46bd0e3cedcc.json deleted file mode 100644 index cef1da340c..0000000000 --- a/backend/.sqlx/query-42030372693d8d6b8d03947bb6702024cfa226735eeeb720bbab46bd0e3cedcc.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT path, is_flow, workspace_id, trigger_config as \"trigger_config!: _\", owner, email FROM capture_config WHERE trigger_kind = 'nats' AND last_client_ping > NOW() - INTERVAL '10 seconds' AND trigger_config IS NOT NULL AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "trigger_config!: _", - "type_info": "Jsonb" - }, - { - "ordinal": 4, - "name": "owner", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "email", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - true, - false, - false - ] - }, - "hash": "42030372693d8d6b8d03947bb6702024cfa226735eeeb720bbab46bd0e3cedcc" -} diff --git a/backend/.sqlx/query-4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc.json b/backend/.sqlx/query-4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc.json deleted file mode 100644 index 6973159764..0000000000 --- a/backend/.sqlx/query-4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "VACUUM v2_job_queue, v2_job_runtime, v2_job_status, job_perms", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc" -} diff --git a/backend/.sqlx/query-42baae3c69367bbb21771a9711c311afe67d339ca022ba61c2767c004b038ef0.json b/backend/.sqlx/query-42baae3c69367bbb21771a9711c311afe67d339ca022ba61c2767c004b038ef0.json new file mode 100644 index 0000000000..de5b99791f --- /dev/null +++ b/backend/.sqlx/query-42baae3c69367bbb21771a9711c311afe67d339ca022ba61c2767c004b038ef0.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "42baae3c69367bbb21771a9711c311afe67d339ca022ba61c2767c004b038ef0" +} diff --git a/backend/.sqlx/query-43c8cd9f8560412bb06d9966ccfa2524943ba277c5f9a37c06bd5ee14fd46bda.json b/backend/.sqlx/query-43c8cd9f8560412bb06d9966ccfa2524943ba277c5f9a37c06bd5ee14fd46bda.json deleted file mode 100644 index 7d6ecdbf32..0000000000 --- a/backend/.sqlx/query-43c8cd9f8560412bb06d9966ccfa2524943ba277c5f9a37c06bd5ee14fd46bda.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'gcp'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "43c8cd9f8560412bb06d9966ccfa2524943ba277c5f9a37c06bd5ee14fd46bda" -} diff --git a/backend/.sqlx/query-44bf04fb504cd1b708657f01c8fb26e81e78807315226557f08449bf43982abb.json b/backend/.sqlx/query-44bf04fb504cd1b708657f01c8fb26e81e78807315226557f08449bf43982abb.json new file mode 100644 index 0000000000..9f0e810e97 --- /dev/null +++ b/backend/.sqlx/query-44bf04fb504cd1b708657f01c8fb26e81e78807315226557f08449bf43982abb.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO debounce_key (key, job_id) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET job_id = EXCLUDED.job_id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "44bf04fb504cd1b708657f01c8fb26e81e78807315226557f08449bf43982abb" +} diff --git a/backend/.sqlx/query-4545e1e0953cde730272353e78044744910a17af23f8201d853f0a4f8404fc73.json b/backend/.sqlx/query-4545e1e0953cde730272353e78044744910a17af23f8201d853f0a4f8404fc73.json new file mode 100644 index 0000000000..c1fd3462f7 --- /dev/null +++ b/backend/.sqlx/query-4545e1e0953cde730272353e78044744910a17af23f8201d853f0a4f8404fc73.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT j.tag as \"tag!\", COUNT(*) as \"count!\"\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE c.started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR j.workspace_id = $2)\n GROUP BY j.tag\n ORDER BY \"count!\" DESC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Float8", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "4545e1e0953cde730272353e78044744910a17af23f8201d853f0a4f8404fc73" +} diff --git a/backend/.sqlx/query-465144ea7e2930203618d9814a3e20c77b4363cf9e7c655d395f3fe40c247f61.json b/backend/.sqlx/query-465144ea7e2930203618d9814a3e20c77b4363cf9e7c655d395f3fe40c247f61.json new file mode 100644 index 0000000000..7869fe789e --- /dev/null +++ b/backend/.sqlx/query-465144ea7e2930203618d9814a3e20c77b4363cf9e7c655d395f3fe40c247f61.json @@ -0,0 +1,74 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n http_trigger\n SET\n route_path = $1,\n route_path_key = $2,\n workspaced_route = $3,\n wrap_body = $4,\n raw_string = $5,\n authentication_resource_path = $6,\n script_path = $7,\n path = $8,\n is_flow = $9,\n http_method = $10,\n static_asset_config = $11,\n edited_by = $12,\n email = $13,\n request_type = $14,\n authentication_method = $15,\n summary = $16,\n description = $17,\n edited_at = now(),\n is_static_website = $18,\n error_handler_path = $19,\n error_handler_args = $20,\n retry = $21\n WHERE\n workspace_id = $22 AND\n path = $23\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + "Bool", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Jsonb", + "Varchar", + "Varchar", + { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + }, + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + "Varchar", + "Text", + "Bool", + "Varchar", + "Jsonb", + "Jsonb", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "465144ea7e2930203618d9814a3e20c77b4363cf9e7c655d395f3fe40c247f61" +} diff --git a/backend/.sqlx/query-3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8.json b/backend/.sqlx/query-469f8b7f691e621cce78b83994b4a4625bb7fbd0974c69745c31c7562563944f.json similarity index 72% rename from backend/.sqlx/query-3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8.json rename to backend/.sqlx/query-469f8b7f691e621cce78b83994b4a4625bb7fbd0974c69745c31c7562563944f.json index 321831d724..e728045afa 100644 --- a/backend/.sqlx/query-3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8.json +++ b/backend/.sqlx/query-469f8b7f691e621cce78b83994b4a4625bb7fbd0974c69745c31c7562563944f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval\n AND running = true AND ping IS NULL AND same_worker = true AND worker IS NOT NULL GROUP BY worker", + "query": "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval\n AND running = true AND (ping IS NULL OR ping < now() - ('60 seconds')::interval) AND same_worker = true AND worker IS NOT NULL GROUP BY worker", "describe": { "columns": [ { @@ -22,5 +22,5 @@ null ] }, - "hash": "3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8" + "hash": "469f8b7f691e621cce78b83994b4a4625bb7fbd0974c69745c31c7562563944f" } diff --git a/backend/.sqlx/query-47560aa3d3af93167663e763ad873088eb4f57785df3c383bbf6e5ceab268980.json b/backend/.sqlx/query-47560aa3d3af93167663e763ad873088eb4f57785df3c383bbf6e5ceab268980.json deleted file mode 100644 index 541634afdf..0000000000 --- a/backend/.sqlx/query-47560aa3d3af93167663e763ad873088eb4f57785df3c383bbf6e5ceab268980.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT path, is_flow, workspace_id, trigger_config as \"trigger_config!: _\", owner, email FROM capture_config WHERE trigger_kind = 'websocket' AND last_client_ping > NOW() - INTERVAL '10 seconds' AND trigger_config IS NOT NULL AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "trigger_config!: _", - "type_info": "Jsonb" - }, - { - "ordinal": 4, - "name": "owner", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "email", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - true, - false, - false - ] - }, - "hash": "47560aa3d3af93167663e763ad873088eb4f57785df3c383bbf6e5ceab268980" -} diff --git a/backend/.sqlx/query-4844a797ddec207f1d2cbf41836e65d6435840f6d0740f300d7c2cf88f8fe46a.json b/backend/.sqlx/query-4844a797ddec207f1d2cbf41836e65d6435840f6d0740f300d7c2cf88f8fe46a.json deleted file mode 100644 index 3f66c4c9b4..0000000000 --- a/backend/.sqlx/query-4844a797ddec207f1d2cbf41836e65d6435840f6d0740f300d7c2cf88f8fe46a.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n mqtt_trigger\n SET \n last_server_ping = now(),\n error = $1\n WHERE\n workspace_id = $2\n AND path = $3\n AND server_id = $4 \n AND enabled IS TRUE\n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "4844a797ddec207f1d2cbf41836e65d6435840f6d0740f300d7c2cf88f8fe46a" -} diff --git a/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json b/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json index af477b4db6..3cfb853bc2 100644 --- a/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json +++ b/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json @@ -37,7 +37,7 @@ "noop", "appdependencies", "deploymentcallback", - "singlescriptflow", + "singlestepflow", "flowscript", "flownode", "appscript", diff --git a/backend/.sqlx/query-7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69.json b/backend/.sqlx/query-48a41e1ad0ad8fc51624af4de34964c72fe4b3cf2497644d514c5c00d6d71bf3.json similarity index 72% rename from backend/.sqlx/query-7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69.json rename to backend/.sqlx/query-48a41e1ad0ad8fc51624af4de34964c72fe4b3cf2497644d514c5c00d6d71bf3.json index e4a0fec1b2..61c93e6573 100644 --- a/backend/.sqlx/query-7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69.json +++ b/backend/.sqlx/query-48a41e1ad0ad8fc51624af4de34964c72fe4b3cf2497644d514c5c00d6d71bf3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n script_path, script_hash AS \"script_hash: ScriptHash\",\n job_kind AS \"job_kind!: JobKind\",\n flow_status AS \"flow_status: Json>\",\n raw_flow AS \"raw_flow: Json>\"\n FROM v2_as_completed_job WHERE id = $1 and workspace_id = $2", + "query": "SELECT\n j.runnable_path as script_path, j.runnable_id AS \"script_hash: ScriptHash\",\n j.kind AS \"job_kind!: JobKind\",\n COALESCE(c.flow_status, c.workflow_as_code_status) AS \"flow_status: Json>\",\n j.raw_flow AS \"raw_flow: Json>\"\n FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.id = $1 and j.workspace_id = $2", "describe": { "columns": [ { @@ -35,7 +35,7 @@ "noop", "appdependencies", "deploymentcallback", - "singlescriptflow", + "singlestepflow", "flowscript", "flownode", "appscript", @@ -65,10 +65,10 @@ "nullable": [ true, true, - true, - true, + false, + null, true ] }, - "hash": "7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69" + "hash": "48a41e1ad0ad8fc51624af4de34964c72fe4b3cf2497644d514c5c00d6d71bf3" } diff --git a/backend/.sqlx/query-2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f.json b/backend/.sqlx/query-49f7e7481019d27b44d5c8a167f9d5d3309a31b195b3147fa32f2ca8a6c9c90e.json similarity index 65% rename from backend/.sqlx/query-2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f.json rename to backend/.sqlx/query-49f7e7481019d27b44d5c8a167f9d5d3309a31b195b3147fa32f2ca8a6c9c90e.json index 07ae8f17c5..d57628b1aa 100644 --- a/backend/.sqlx/query-2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f.json +++ b/backend/.sqlx/query-49f7e7481019d27b44d5c8a167f9d5d3309a31b195b3147fa32f2ca8a6c9c90e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n script_path, args AS \"args: sqlx::types::Json>>\",\n tag AS \"tag!\", priority\n FROM v2_as_completed_job\n WHERE id = $1 and workspace_id = $2", + "query": "SELECT\n j.runnable_path as script_path, j.args AS \"args: sqlx::types::Json>>\",\n j.tag AS \"tag!\", j.priority\n FROM v2_job j\n WHERE j.id = $1 and j.workspace_id = $2", "describe": { "columns": [ { @@ -33,9 +33,9 @@ "nullable": [ true, true, - true, + false, true ] }, - "hash": "2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f" + "hash": "49f7e7481019d27b44d5c8a167f9d5d3309a31b195b3147fa32f2ca8a6c9c90e" } diff --git a/backend/.sqlx/query-4b056d33215b3a1e9849bb66ce84e96c69d10e3970e38151c97d1fca0cb7388d.json b/backend/.sqlx/query-4b056d33215b3a1e9849bb66ce84e96c69d10e3970e38151c97d1fca0cb7388d.json new file mode 100644 index 0000000000..da7115eb81 --- /dev/null +++ b/backend/.sqlx/query-4b056d33215b3a1e9849bb66ce84e96c69d10e3970e38151c97d1fca0cb7388d.json @@ -0,0 +1,49 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH result_stream AS (\n SELECT \n string_agg(stream, '' order by idx asc) as stream, \n job_id, \n max(idx) + 1 as offset \n FROM job_result_stream_v2\n WHERE job_id = $2 AND idx >= $3\n GROUP BY job_id\n )\n SELECT\n COALESCE(jc.result, NULL) as \"result: sqlx::types::Json>\",\n rs.stream AS \"result_stream: Option\",\n rs.offset AS stream_offset,\n COALESCE(js.flow_status, jc.flow_status) as \"flow_status: sqlx::types::Json>\",\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM (\n SELECT $2::uuid as job_id, $1::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN v2_job_status js ON js.id = base.job_id\n LEFT JOIN result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "result_stream: Option", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "stream_offset", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "flow_status: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "stream_job", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Int4", + "Bool" + ] + }, + "nullable": [ + null, + null, + null, + null, + null + ] + }, + "hash": "4b056d33215b3a1e9849bb66ce84e96c69d10e3970e38151c97d1fca0cb7388d" +} diff --git a/backend/.sqlx/query-4bc533074c720820cebff8d97a203df52520b7606378ecca267e88383a45b49b.json b/backend/.sqlx/query-4bc533074c720820cebff8d97a203df52520b7606378ecca267e88383a45b49b.json deleted file mode 100644 index 8c9de521d2..0000000000 --- a/backend/.sqlx/query-4bc533074c720820cebff8d97a203df52520b7606378ecca267e88383a45b49b.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO job_result_stream (workspace_id, job_id, stream)\n VALUES ($1, $2, $3)\n ON CONFLICT (job_id) DO UPDATE SET stream = job_result_stream.stream || $3\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text" - ] - }, - "nullable": [] - }, - "hash": "4bc533074c720820cebff8d97a203df52520b7606378ecca267e88383a45b49b" -} diff --git a/backend/.sqlx/query-4c64bd0e364f536597db83161b5a27ff58b5ec7148bc94807423593bcfa27b90.json b/backend/.sqlx/query-4c64bd0e364f536597db83161b5a27ff58b5ec7148bc94807423593bcfa27b90.json new file mode 100644 index 0000000000..dfe905113e --- /dev/null +++ b/backend/.sqlx/query-4c64bd0e364f536597db83161b5a27ff58b5ec7148bc94807423593bcfa27b90.json @@ -0,0 +1,71 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n http_trigger\n SET\n wrap_body = $1,\n raw_string = $2,\n authentication_resource_path = $3,\n script_path = $4,\n path = $5,\n is_flow = $6,\n http_method = $7,\n static_asset_config = $8,\n edited_by = $9,\n email = $10,\n request_type = $11,\n authentication_method = $12,\n summary = $13,\n description = $14,\n edited_at = now(),\n is_static_website = $15,\n error_handler_path = $16,\n error_handler_args = $17,\n retry = $18\n WHERE\n workspace_id = $19 AND\n path = $20\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Jsonb", + "Varchar", + "Varchar", + { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + }, + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + "Varchar", + "Text", + "Bool", + "Varchar", + "Jsonb", + "Jsonb", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4c64bd0e364f536597db83161b5a27ff58b5ec7148bc94807423593bcfa27b90" +} diff --git a/backend/.sqlx/query-9a9e4a8779b0bf8a275d029221dfa1465e5d44cd8a7be5879219ffc8cd7ae6b1.json b/backend/.sqlx/query-4d3b8726656ebf6eb78d0c4c23fdfa6325fcae4dc3bf559f2ecc683fb40537e8.json similarity index 69% rename from backend/.sqlx/query-9a9e4a8779b0bf8a275d029221dfa1465e5d44cd8a7be5879219ffc8cd7ae6b1.json rename to backend/.sqlx/query-4d3b8726656ebf6eb78d0c4c23fdfa6325fcae4dc3bf559f2ecc683fb40537e8.json index 4a85852957..83c02ba868 100644 --- a/backend/.sqlx/query-9a9e4a8779b0bf8a275d029221dfa1465e5d44cd8a7be5879219ffc8cd7ae6b1.json +++ b/backend/.sqlx/query-4d3b8726656ebf6eb78d0c4c23fdfa6325fcae4dc3bf559f2ecc683fb40537e8.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('5 mins')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", + "query": "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('5 mins')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = EXCLUDED.lockfile", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "9a9e4a8779b0bf8a275d029221dfa1465e5d44cd8a7be5879219ffc8cd7ae6b1" + "hash": "4d3b8726656ebf6eb78d0c4c23fdfa6325fcae4dc3bf559f2ecc683fb40537e8" } diff --git a/backend/.sqlx/query-4f11760f5d283728ded5533e6a0b51f49fdb7c11bf7d47b8536d607e646bd265.json b/backend/.sqlx/query-4f11760f5d283728ded5533e6a0b51f49fdb7c11bf7d47b8536d607e646bd265.json new file mode 100644 index 0000000000..40c82a50aa --- /dev/null +++ b/backend/.sqlx/query-4f11760f5d283728ded5533e6a0b51f49fdb7c11bf7d47b8536d607e646bd265.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id FROM debounce_key WHERE key = $1 AND job_id IN (SELECT id FROM v2_job_queue) FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "4f11760f5d283728ded5533e6a0b51f49fdb7c11bf7d47b8536d607e646bd265" +} diff --git a/backend/.sqlx/query-4fcce9b5b039b73b2f19fa9d15294bbb30311749431f31e488c32d21b2337544.json b/backend/.sqlx/query-4fcce9b5b039b73b2f19fa9d15294bbb30311749431f31e488c32d21b2337544.json new file mode 100644 index 0000000000..4c108cfea2 --- /dev/null +++ b/backend/.sqlx/query-4fcce9b5b039b73b2f19fa9d15294bbb30311749431f31e488c32d21b2337544.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (\n workspace_id, hash, path, parent_hashes, summary, description, content,\n created_by, created_at, archived, schema, deleted, is_template,\n extra_perms, lock, lock_error_logs, language, kind, tag, draft_only,\n envs, concurrent_limit, concurrency_time_window_s, cache_ttl,\n dedicated_worker, ws_error_handler_muted, priority, timeout,\n delete_after_use, restart_unless_cancelled, concurrency_key,\n visible_to_runner_only, no_main_func, codebase, has_preprocessor,\n on_behalf_of_email, assets\n )\n SELECT\n $1, hash, path, parent_hashes, summary, description, content,\n created_by, created_at, archived, schema, deleted, is_template,\n extra_perms, lock, lock_error_logs, language, kind, tag, draft_only,\n envs, concurrent_limit, concurrency_time_window_s, cache_ttl,\n dedicated_worker, ws_error_handler_muted, priority, timeout,\n delete_after_use, restart_unless_cancelled, concurrency_key,\n visible_to_runner_only, no_main_func, codebase, has_preprocessor,\n on_behalf_of_email, assets\n FROM script\n WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4fcce9b5b039b73b2f19fa9d15294bbb30311749431f31e488c32d21b2337544" +} diff --git a/backend/.sqlx/query-50261db5d492cb9ba56e6ebc3eb4628b45a3c2ed6a6eb65ac8815992d5114e70.json b/backend/.sqlx/query-50261db5d492cb9ba56e6ebc3eb4628b45a3c2ed6a6eb65ac8815992d5114e70.json new file mode 100644 index 0000000000..31f9825cda --- /dev/null +++ b/backend/.sqlx/query-50261db5d492cb9ba56e6ebc3eb4628b45a3c2ed6a6eb65ac8815992d5114e70.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT fv.value->>'early_return' as \"early_return\"\n FROM v2_job j\n INNER JOIN flow_version fv ON fv.id = j.runnable_id\n WHERE j.id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "early_return", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "50261db5d492cb9ba56e6ebc3eb4628b45a3c2ed6a6eb65ac8815992d5114e70" +} diff --git a/backend/.sqlx/query-50c17c7848760aaf0f869acfc444caecda6335eda5b2e97e5a7370361653ff48.json b/backend/.sqlx/query-50c17c7848760aaf0f869acfc444caecda6335eda5b2e97e5a7370361653ff48.json deleted file mode 100644 index f3dc153254..0000000000 --- a/backend/.sqlx/query-50c17c7848760aaf0f869acfc444caecda6335eda5b2e97e5a7370361653ff48.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH assigned_teams AS (\n SELECT teams_team_id\n FROM workspace_settings\n ),\n all_teams AS (\n SELECT jsonb_array_elements(CASE\n WHEN jsonb_typeof(value::jsonb) = 'array' THEN value::jsonb\n ELSE '[]'::jsonb\n END) AS team\n FROM global_settings\n WHERE name = 'teams'\n )\n SELECT team->>'team_name' AS team_name, team->>'team_internal_id' AS team_id\n FROM all_teams\n WHERE NOT EXISTS (\n SELECT 1\n FROM assigned_teams\n WHERE assigned_teams.teams_team_id = team->>'team_internal_id'\n )\n AND team->>'team_name' IS NOT NULL\n AND team->>'team_id' IS NOT NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "team_name", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "team_id", - "type_info": "Text" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - null - ] - }, - "hash": "50c17c7848760aaf0f869acfc444caecda6335eda5b2e97e5a7370361653ff48" -} diff --git a/backend/.sqlx/query-519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb.json b/backend/.sqlx/query-519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb.json deleted file mode 100644 index b2c0b4702c..0000000000 --- a/backend/.sqlx/query-519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM v2_as_queue WHERE canceled = false AND (scheduled_for <= now()\n OR (suspend_until IS NOT NULL\n AND ( suspend <= 0\n OR suspend_until <= now())))", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb" -} diff --git a/backend/.sqlx/query-51f37d683d5a48b96f6224111639c364ba9c41572eb6799f7c822c66d00d2500.json b/backend/.sqlx/query-51f37d683d5a48b96f6224111639c364ba9c41572eb6799f7c822c66d00d2500.json deleted file mode 100644 index 36198820c2..0000000000 --- a/backend/.sqlx/query-51f37d683d5a48b96f6224111639c364ba9c41572eb6799f7c822c66d00d2500.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE websocket_trigger SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "51f37d683d5a48b96f6224111639c364ba9c41572eb6799f7c822c66d00d2500" -} diff --git a/backend/.sqlx/query-528cdbb75f1c5135170a58fce3fda464be138272487639d0ffbbbe6961ec5c37.json b/backend/.sqlx/query-528cdbb75f1c5135170a58fce3fda464be138272487639d0ffbbbe6961ec5c37.json deleted file mode 100644 index e998bea512..0000000000 --- a/backend/.sqlx/query-528cdbb75f1c5135170a58fce3fda464be138272487639d0ffbbbe6961ec5c37.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE job_logs SET logs = $1, log_offset = $2, \n log_file_index = array_append(coalesce(log_file_index, array[]::text[]), $3) \n WHERE workspace_id = $4 AND job_id = $5", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int4", - "Text", - "Text", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "528cdbb75f1c5135170a58fce3fda464be138272487639d0ffbbbe6961ec5c37" -} diff --git a/backend/.sqlx/query-529a52823913f4154786a2ada93f6112c39575a52a1bfc02f27d9b1185b0578e.json b/backend/.sqlx/query-529a52823913f4154786a2ada93f6112c39575a52a1bfc02f27d9b1185b0578e.json new file mode 100644 index 0000000000..732ae64ad6 --- /dev/null +++ b/backend/.sqlx/query-529a52823913f4154786a2ada93f6112c39575a52a1bfc02f27d9b1185b0578e.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\nDELETE FROM debounce_key\nWHERE job_id NOT IN (SELECT id FROM v2_job_queue)\nRETURNING key,job_id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "key", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false + ] + }, + "hash": "529a52823913f4154786a2ada93f6112c39575a52a1bfc02f27d9b1185b0578e" +} diff --git a/backend/.sqlx/query-53210d19693ba46d97352b0ee321af59db453243c0cd17928ee942723d9cdc5c.json b/backend/.sqlx/query-53210d19693ba46d97352b0ee321af59db453243c0cd17928ee942723d9cdc5c.json new file mode 100644 index 0000000000..dfb445ed36 --- /dev/null +++ b/backend/.sqlx/query-53210d19693ba46d97352b0ee321af59db453243c0cd17928ee942723d9cdc5c.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, auto_add_instance_groups_roles FROM workspace_settings WHERE $1 = ANY(COALESCE(auto_add_instance_groups, '{}'))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "auto_add_instance_groups_roles", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "53210d19693ba46d97352b0ee321af59db453243c0cd17928ee942723d9cdc5c" +} diff --git a/backend/.sqlx/query-53dee7c119d724624b9973ee981576154cec84a09069286d2d7144dbad54f4d6.json b/backend/.sqlx/query-53dee7c119d724624b9973ee981576154cec84a09069286d2d7144dbad54f4d6.json new file mode 100644 index 0000000000..902ed458c2 --- /dev/null +++ b/backend/.sqlx/query-53dee7c119d724624b9973ee981576154cec84a09069286d2d7144dbad54f4d6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "53dee7c119d724624b9973ee981576154cec84a09069286d2d7144dbad54f4d6" +} diff --git a/backend/.sqlx/query-548a1424a6b9ac9998b8fc5312bcff671ee7fe59154e9ef32800c117e64bce19.json b/backend/.sqlx/query-548a1424a6b9ac9998b8fc5312bcff671ee7fe59154e9ef32800c117e64bce19.json deleted file mode 100644 index 554b0fff0b..0000000000 --- a/backend/.sqlx/query-548a1424a6b9ac9998b8fc5312bcff671ee7fe59154e9ef32800c117e64bce19.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n path,\n is_flow,\n workspace_id,\n owner,\n email,\n trigger_config as \"trigger_config!: _\"\n FROM\n capture_config\n WHERE\n trigger_kind = 'mqtt' AND\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n trigger_config IS NOT NULL AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "owner", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "trigger_config!: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - false, - true - ] - }, - "hash": "548a1424a6b9ac9998b8fc5312bcff671ee7fe59154e9ef32800c117e64bce19" -} diff --git a/backend/.sqlx/query-54b7d6614bdfae5d9113c1baf9eae6e1f600c0593ba855138b435561687905c5.json b/backend/.sqlx/query-54b7d6614bdfae5d9113c1baf9eae6e1f600c0593ba855138b435561687905c5.json new file mode 100644 index 0000000000..7657331ba1 --- /dev/null +++ b/backend/.sqlx/query-54b7d6614bdfae5d9113c1baf9eae6e1f600c0593ba855138b435561687905c5.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\nSELECT COALESCE(\n (SELECT COUNT(*) \n FROM jsonb_object_keys(job_uuids) AS keys(key) \n WHERE key <> $2),\n 0\n)\nFROM concurrency_counter\nWHERE concurrency_id = $1\nFOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "coalesce", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "54b7d6614bdfae5d9113c1baf9eae6e1f600c0593ba855138b435561687905c5" +} diff --git a/backend/.sqlx/query-c7dd35561e9b1cfd86238d410139f50e4d87c762e4867d8b32c11bd8c74846eb.json b/backend/.sqlx/query-5585bc46fe2b9d5aebddad300f43b7cbc891e55d29d30d1ca7b13f25ed1fdc87.json similarity index 74% rename from backend/.sqlx/query-c7dd35561e9b1cfd86238d410139f50e4d87c762e4867d8b32c11bd8c74846eb.json rename to backend/.sqlx/query-5585bc46fe2b9d5aebddad300f43b7cbc891e55d29d30d1ca7b13f25ed1fdc87.json index 73a1911081..0df8109669 100644 --- a/backend/.sqlx/query-c7dd35561e9b1cfd86238d410139f50e4d87c762e4867d8b32c11bd8c74846eb.json +++ b/backend/.sqlx/query-5585bc46fe2b9d5aebddad300f43b7cbc891e55d29d30d1ca7b13f25ed1fdc87.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, $1::text)", + "query": "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, EXCLUDED.logs)", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "c7dd35561e9b1cfd86238d410139f50e4d87c762e4867d8b32c11bd8c74846eb" + "hash": "5585bc46fe2b9d5aebddad300f43b7cbc891e55d29d30d1ca7b13f25ed1fdc87" } diff --git a/backend/.sqlx/query-56b2326015fde12b1a4efa226518566101dd27a0f3363884781071d417f8b7e7.json b/backend/.sqlx/query-56b2326015fde12b1a4efa226518566101dd27a0f3363884781071d417f8b7e7.json deleted file mode 100644 index 9da0554266..0000000000 --- a/backend/.sqlx/query-56b2326015fde12b1a4efa226518566101dd27a0f3363884781071d417f8b7e7.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 \n RETURNING usage.usage", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "usage", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar" - ] - }, - "nullable": [ - false - ] - }, - "hash": "56b2326015fde12b1a4efa226518566101dd27a0f3363884781071d417f8b7e7" -} diff --git a/backend/.sqlx/query-08dd2ea6b17a52bce352d6443d7d009cfc9da0d3b2bd1f40d422b550779e5324.json b/backend/.sqlx/query-5701ee0b862dbdb44990702af270a2eb517e82943bb8c078c7bb2e60def3cbf1.json similarity index 72% rename from backend/.sqlx/query-08dd2ea6b17a52bce352d6443d7d009cfc9da0d3b2bd1f40d422b550779e5324.json rename to backend/.sqlx/query-5701ee0b862dbdb44990702af270a2eb517e82943bb8c078c7bb2e60def3cbf1.json index b56784fd64..e1384a4e70 100644 --- a/backend/.sqlx/query-08dd2ea6b17a52bce352d6443d7d009cfc9da0d3b2bd1f40d422b550779e5324.json +++ b/backend/.sqlx/query-5701ee0b862dbdb44990702af270a2eb517e82943bb8c078c7bb2e60def3cbf1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO variable\n (workspace_id, path, value, is_secret, account, is_oauth, expires_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n ON CONFLICT (workspace_id, path) DO UPDATE SET value = $3, expires_at = $7", + "query": "INSERT INTO variable\n (workspace_id, path, value, is_secret, account, is_oauth, expires_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n ON CONFLICT (workspace_id, path) DO UPDATE SET value = EXCLUDED.value, expires_at = EXCLUDED.expires_at", "describe": { "columns": [], "parameters": { @@ -16,5 +16,5 @@ }, "nullable": [] }, - "hash": "08dd2ea6b17a52bce352d6443d7d009cfc9da0d3b2bd1f40d422b550779e5324" + "hash": "5701ee0b862dbdb44990702af270a2eb517e82943bb8c078c7bb2e60def3cbf1" } diff --git a/backend/.sqlx/query-0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd.json b/backend/.sqlx/query-57bd445bfdc667089f847acb9dc484028ce8a077308979a5ba8ef252e19aa825.json similarity index 50% rename from backend/.sqlx/query-0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd.json rename to backend/.sqlx/query-57bd445bfdc667089f847acb9dc484028ce8a077308979a5ba8ef252e19aa825.json index 0a9e91b206..1bb262acbf 100644 --- a/backend/.sqlx/query-0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd.json +++ b/backend/.sqlx/query-57bd445bfdc667089f847acb9dc484028ce8a077308979a5ba8ef252e19aa825.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT coalesce(COUNT(*) FILTER(WHERE suspend = 0 AND running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE suspend > 0), 0) as \"suspended!\" FROM v2_as_queue WHERE (workspace_id = $1 OR $2) AND scheduled_for <= now() AND ($3::text[] IS NULL OR tag = ANY($3))", + "query": "SELECT coalesce(COUNT(*) FILTER(WHERE q.suspend = 0 AND q.running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE q.suspend > 0), 0) as \"suspended!\" FROM v2_job_queue q JOIN v2_job j USING (id) WHERE (j.workspace_id = $1 OR $2) AND q.scheduled_for <= now() AND ($3::text[] IS NULL OR j.tag = ANY($3))", "describe": { "columns": [ { @@ -26,5 +26,5 @@ null ] }, - "hash": "0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd" + "hash": "57bd445bfdc667089f847acb9dc484028ce8a077308979a5ba8ef252e19aa825" } diff --git a/backend/.sqlx/query-58ddf5c76455d30dc42f46a0553e761461c327e7502b707c2d2742946d7d1f4c.json b/backend/.sqlx/query-58ddf5c76455d30dc42f46a0553e761461c327e7502b707c2d2742946d7d1f4c.json deleted file mode 100644 index 2a91229fbd..0000000000 --- a/backend/.sqlx/query-58ddf5c76455d30dc42f46a0553e761461c327e7502b707c2d2742946d7d1f4c.json +++ /dev/null @@ -1,138 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n aws_auth_resource_type AS \"aws_auth_resource_type: _\",\n queue_url,\n aws_resource_path,\n message_attributes,\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM\n sqs_trigger\n WHERE\n enabled IS TRUE\n AND (last_server_ping IS NULL OR\n last_server_ping < now() - interval '15 seconds'\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "aws_auth_resource_type: _", - "type_info": { - "Custom": { - "name": "aws_auth_resource_type", - "kind": { - "Enum": [ - "oidc", - "credentials" - ] - } - } - } - }, - { - "ordinal": 1, - "name": "queue_url", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "aws_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "message_attributes", - "type_info": "TextArray" - }, - { - "ordinal": 4, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 8, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 10, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 11, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 12, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 13, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 14, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 15, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 16, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 17, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 18, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - true, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - true, - false, - true, - true, - true - ] - }, - "hash": "58ddf5c76455d30dc42f46a0553e761461c327e7502b707c2d2742946d7d1f4c" -} diff --git a/backend/.sqlx/query-59c63c5e3ce0b4976133cd65258ef6bfdecc81700b89962c758c065d8d55f9e2.json b/backend/.sqlx/query-59c63c5e3ce0b4976133cd65258ef6bfdecc81700b89962c758c065d8d55f9e2.json new file mode 100644 index 0000000000..f2c1cd78bf --- /dev/null +++ b/backend/.sqlx/query-59c63c5e3ce0b4976133cd65258ef6bfdecc81700b89962c758c065d8d55f9e2.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO instance_group (name, summary, id) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "59c63c5e3ce0b4976133cd65258ef6bfdecc81700b89962c758c065d8d55f9e2" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-0fa105c49c8345916716514444bd3616ae4d114216c659233fbbc3c047e6b30a.json b/backend/.sqlx/query-5ad2c883d26f39f3c141806428a329951ef19a1cb3e1b429fcd1abe0e2db45b5.json similarity index 65% rename from backend/.sqlx/query-0fa105c49c8345916716514444bd3616ae4d114216c659233fbbc3c047e6b30a.json rename to backend/.sqlx/query-5ad2c883d26f39f3c141806428a329951ef19a1cb3e1b429fcd1abe0e2db45b5.json index c604c19f04..4b7a7ab231 100644 --- a/backend/.sqlx/query-0fa105c49c8345916716514444bd3616ae4d114216c659233fbbc3c047e6b30a.json +++ b/backend/.sqlx/query-5ad2c883d26f39f3c141806428a329951ef19a1cb3e1b429fcd1abe0e2db45b5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE\n workspace_settings\n SET\n error_handler = NULL,\n error_handler_extra_args = NULL,\n error_handler_muted_on_cancel = NULL\n WHERE\n workspace_id = $1\n ", + "query": "\n UPDATE\n workspace_settings\n SET\n error_handler = NULL,\n error_handler_extra_args = NULL,\n error_handler_muted_on_cancel = false\n WHERE\n workspace_id = $1\n ", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "0fa105c49c8345916716514444bd3616ae4d114216c659233fbbc3c047e6b30a" + "hash": "5ad2c883d26f39f3c141806428a329951ef19a1cb3e1b429fcd1abe0e2db45b5" } diff --git a/backend/.sqlx/query-b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46.json b/backend/.sqlx/query-5adeb6989648ca42431c96069c4de88d2615e7e6f1267f2fb12ccc325d4e4148.json similarity index 51% rename from backend/.sqlx/query-b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46.json rename to backend/.sqlx/query-5adeb6989648ca42431c96069c4de88d2615e7e6f1267f2fb12ccc325d4e4148.json index a49baeefaf..859f2b23a4 100644 --- a/backend/.sqlx/query-b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46.json +++ b/backend/.sqlx/query-5adeb6989648ca42431c96069c4de88d2615e7e6f1267f2fb12ccc325d4e4148.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_runtime (id) SELECT unnest($1::uuid[])", + "query": "DELETE FROM job_result_stream_v2 WHERE job_id = ANY($1)", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46" + "hash": "5adeb6989648ca42431c96069c4de88d2615e7e6f1267f2fb12ccc325d4e4148" } diff --git a/backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json b/backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json deleted file mode 100644 index 31700319e2..0000000000 --- a/backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n sqs_trigger \n SET \n server_id = $1, \n last_server_ping = now(),\n error = 'Connecting...'\n WHERE \n enabled IS TRUE \n AND workspace_id = $2 \n AND path = $3 \n AND (last_server_ping IS NULL \n OR last_server_ping < now() - INTERVAL '15 seconds'\n ) \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71" -} diff --git a/backend/.sqlx/query-5c13c681df0c57fcdbd1364f0f084aad096911efd453dd50f6865e53c1e77881.json b/backend/.sqlx/query-5c13c681df0c57fcdbd1364f0f084aad096911efd453dd50f6865e53c1e77881.json new file mode 100644 index 0000000000..4d756dbaa7 --- /dev/null +++ b/backend/.sqlx/query-5c13c681df0c57fcdbd1364f0f084aad096911efd453dd50f6865e53c1e77881.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n flow_step_id,\n (flow_status->'step')::integer as step,\n jsonb_array_length(flow_status->'modules') as len,\n runnable_path ~ '/branchone-\\d+$' as is_branch_one,\n parent_job as next_parent\n FROM v2_job\n LEFT JOIN v2_job_status USING (id)\n WHERE v2_job.id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "step", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "len", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "is_branch_one", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "next_parent", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true, + null, + null, + null, + true + ] + }, + "hash": "5c13c681df0c57fcdbd1364f0f084aad096911efd453dd50f6865e53c1e77881" +} diff --git a/backend/.sqlx/query-5c403799772e1c44bd00e5376893a6428b3c7cbdef4af70a6660c1a5900d17a6.json b/backend/.sqlx/query-5c403799772e1c44bd00e5376893a6428b3c7cbdef4af70a6660c1a5900d17a6.json new file mode 100644 index 0000000000..5dc78dab56 --- /dev/null +++ b/backend/.sqlx/query-5c403799772e1c44bd00e5376893a6428b3c7cbdef4af70a6660c1a5900d17a6.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "create index concurrently if not exists ix_job_workspace_id_completed_at_all ON v2_job_completed (workspace_id, completed_at DESC)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "5c403799772e1c44bd00e5376893a6428b3c7cbdef4af70a6660c1a5900d17a6" +} diff --git a/backend/.sqlx/query-5a9deb187b43fde22c4d32629bc030d2fe256d647a0422ba40dfe15a513dc04d.json b/backend/.sqlx/query-5c72e6e4102039c4426e8eb47d86ec152d4ee52b58f208f2fa1639eaa448c6ad.json similarity index 64% rename from backend/.sqlx/query-5a9deb187b43fde22c4d32629bc030d2fe256d647a0422ba40dfe15a513dc04d.json rename to backend/.sqlx/query-5c72e6e4102039c4426e8eb47d86ec152d4ee52b58f208f2fa1639eaa448c6ad.json index 7aef978d6e..3458d1068e 100644 --- a/backend/.sqlx/query-5a9deb187b43fde22c4d32629bc030d2fe256d647a0422ba40dfe15a513dc04d.json +++ b/backend/.sqlx/query-5c72e6e4102039c4426e8eb47d86ec152d4ee52b58f208f2fa1639eaa448c6ad.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO app_script (app, hash, lock, code, code_sha256)\n VALUES ($1, $2, $3, $4, $5)", + "query": "INSERT INTO app_script (app, hash, lock, code, code_sha256)\n VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", "describe": { "columns": [], "parameters": { @@ -14,5 +14,5 @@ }, "nullable": [] }, - "hash": "5a9deb187b43fde22c4d32629bc030d2fe256d647a0422ba40dfe15a513dc04d" + "hash": "5c72e6e4102039c4426e8eb47d86ec152d4ee52b58f208f2fa1639eaa448c6ad" } diff --git a/backend/.sqlx/query-5cc0d9e3dcd9c20e6e6ec1acf38c8f97b5ece60bfe24fc3783a83de47e3aa583.json b/backend/.sqlx/query-5cc0d9e3dcd9c20e6e6ec1acf38c8f97b5ece60bfe24fc3783a83de47e3aa583.json deleted file mode 100644 index fc2540e85d..0000000000 --- a/backend/.sqlx/query-5cc0d9e3dcd9c20e6e6ec1acf38c8f97b5ece60bfe24fc3783a83de47e3aa583.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO http_trigger (\n workspace_id, \n path, \n route_path, \n route_path_key,\n workspaced_route,\n authentication_resource_path,\n wrap_body,\n raw_string,\n script_path, \n summary,\n description,\n is_flow, \n is_async, \n authentication_method, \n http_method, \n static_asset_config, \n edited_by, \n email, \n edited_at, \n is_static_website,\n error_handler_path,\n error_handler_args,\n retry\n ) \n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, now(), $19, $20, $21, $22\n )\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Bool", - "Bool", - "Varchar", - "Varchar", - "Text", - "Bool", - "Bool", - { - "Custom": { - "name": "authentication_method", - "kind": { - "Enum": [ - "none", - "windmill", - "api_key", - "basic_http", - "custom_script", - "signature" - ] - } - } - }, - { - "Custom": { - "name": "http_method", - "kind": { - "Enum": [ - "get", - "post", - "put", - "delete", - "patch" - ] - } - } - }, - "Jsonb", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "5cc0d9e3dcd9c20e6e6ec1acf38c8f97b5ece60bfe24fc3783a83de47e3aa583" -} diff --git a/backend/.sqlx/query-d585aa6301c41308b02a1f0fbf068221e732e48dfa6e34d5b025adbbdcbb03e0.json b/backend/.sqlx/query-5d99d2b058d4896f9ac1cea04fa35c003bc4e897ee01746d6f17aeee387d1505.json similarity index 62% rename from backend/.sqlx/query-d585aa6301c41308b02a1f0fbf068221e732e48dfa6e34d5b025adbbdcbb03e0.json rename to backend/.sqlx/query-5d99d2b058d4896f9ac1cea04fa35c003bc4e897ee01746d6f17aeee387d1505.json index 72169ec94b..c22e4bc5cf 100644 --- a/backend/.sqlx/query-d585aa6301c41308b02a1f0fbf068221e732e48dfa6e34d5b025adbbdcbb03e0.json +++ b/backend/.sqlx/query-5d99d2b058d4896f9ac1cea04fa35c003bc4e897ee01746d6f17aeee387d1505.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "create index concurrently if not exists ix_v2_job_workspace_id_created_at ON v2_job (workspace_id, created_at DESC) where kind in ('script', 'flow', 'singlescriptflow') AND parent_job IS NULL", + "query": "create index concurrently if not exists ix_v2_job_workspace_id_created_at ON v2_job (workspace_id, created_at DESC) where kind in ('script', 'flow', 'singlestepflow') AND parent_job IS NULL", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "d585aa6301c41308b02a1f0fbf068221e732e48dfa6e34d5b025adbbdcbb03e0" + "hash": "5d99d2b058d4896f9ac1cea04fa35c003bc4e897ee01746d6f17aeee387d1505" } diff --git a/backend/.sqlx/query-430ea56dea72c7d326735952bf85708dcc4811857b7e5631f0decf5d75ef3aa3.json b/backend/.sqlx/query-5e6dcb3e7a9bc174a040cfba96555eb5140c563a2b191714ef9028b56391b794.json similarity index 77% rename from backend/.sqlx/query-430ea56dea72c7d326735952bf85708dcc4811857b7e5631f0decf5d75ef3aa3.json rename to backend/.sqlx/query-5e6dcb3e7a9bc174a040cfba96555eb5140c563a2b191714ef9028b56391b794.json index 660b5cb402..5863dc5f98 100644 --- a/backend/.sqlx/query-430ea56dea72c7d326735952bf85708dcc4811857b7e5631f0decf5d75ef3aa3.json +++ b/backend/.sqlx/query-5e6dcb3e7a9bc174a040cfba96555eb5140c563a2b191714ef9028b56391b794.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n schedule.path, t.jobs FROM schedule,\n LATERAL(SELECT ARRAY(\n SELECT json_build_object('id', id, 'success', status = 'success', 'duration_ms', duration_ms)\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE trigger_kind = 'schedule'\n AND trigger = schedule.path\n AND c.workspace_id = $1\n AND j.workspace_id = $1\n AND parent_job IS NULL AND runnable_path = schedule.script_path\n AND status <> 'skipped'\n ORDER BY created_at DESC\n LIMIT 20\n ) AS jobs) t\n WHERE workspace_id = $1\n ORDER BY edited_at DESC\n LIMIT $2 OFFSET $3", + "query": "SELECT\n schedule.path, t.jobs FROM schedule,\n LATERAL(SELECT ARRAY(\n SELECT json_build_object('id', id, 'success', status = 'success', 'duration_ms', duration_ms)\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE trigger_kind = 'schedule'\n AND trigger = schedule.path\n AND c.workspace_id = $1\n AND j.workspace_id = $1\n AND parent_job IS NULL AND runnable_path = schedule.script_path\n AND status <> 'skipped'\n ORDER BY completed_at DESC\n LIMIT 20\n ) AS jobs) t\n WHERE workspace_id = $1\n ORDER BY edited_at DESC\n LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -26,5 +26,5 @@ null ] }, - "hash": "430ea56dea72c7d326735952bf85708dcc4811857b7e5631f0decf5d75ef3aa3" + "hash": "5e6dcb3e7a9bc174a040cfba96555eb5140c563a2b191714ef9028b56391b794" } diff --git a/backend/.sqlx/query-a22e166746ee10943c737668aafd548e3adefe5847d2e35331edb61442d0dd92.json b/backend/.sqlx/query-5fdfc9427f455a4c1bc8f6ca41ddfd426bc0c2ac126792c926f3cf1182ded981.json similarity index 50% rename from backend/.sqlx/query-a22e166746ee10943c737668aafd548e3adefe5847d2e35331edb61442d0dd92.json rename to backend/.sqlx/query-5fdfc9427f455a4c1bc8f6ca41ddfd426bc0c2ac126792c926f3cf1182ded981.json index 33005ff7a6..41f7673e92 100644 --- a/backend/.sqlx/query-a22e166746ee10943c737668aafd548e3adefe5847d2e35331edb61442d0dd92.json +++ b/backend/.sqlx/query-5fdfc9427f455a4c1bc8f6ca41ddfd426bc0c2ac126792c926f3cf1182ded981.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT trim(both '\"' from value::text) FROM global_settings WHERE name = 'ducklake_user_pg_pwd';", + "query": "SELECT value->>'ducklake_user_pg_pwd' FROM global_settings WHERE name = 'ducklake_settings';", "describe": { "columns": [ { "ordinal": 0, - "name": "btrim", + "name": "?column?", "type_info": "Text" } ], @@ -16,5 +16,5 @@ null ] }, - "hash": "a22e166746ee10943c737668aafd548e3adefe5847d2e35331edb61442d0dd92" + "hash": "5fdfc9427f455a4c1bc8f6ca41ddfd426bc0c2ac126792c926f3cf1182ded981" } diff --git a/backend/.sqlx/query-615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae.json b/backend/.sqlx/query-615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae.json new file mode 100644 index 0000000000..da8704fe30 --- /dev/null +++ b/backend/.sqlx/query-615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE \n capture_config\n SET \n last_server_ping = now(), error = $1\n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = $5 AND \n server_id = $6 AND \n last_client_ping > NOW() - INTERVAL '10 seconds'\n RETURNING 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae" +} diff --git a/backend/.sqlx/query-61bed1bc6d3e6a3c1d640eeacc290a85d8b63ee36c39dfbf4348d120f6e561ae.json b/backend/.sqlx/query-61bed1bc6d3e6a3c1d640eeacc290a85d8b63ee36c39dfbf4348d120f6e561ae.json deleted file mode 100644 index 7f9886ccb6..0000000000 --- a/backend/.sqlx/query-61bed1bc6d3e6a3c1d640eeacc290a85d8b63ee36c39dfbf4348d120f6e561ae.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n postgres_trigger \n SET\n last_server_ping = NULL \n WHERE \n workspace_id = $1 \n AND path = $2 \n AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "61bed1bc6d3e6a3c1d640eeacc290a85d8b63ee36c39dfbf4348d120f6e561ae" -} diff --git a/backend/.sqlx/query-668edc2f84eccf5db6b8daa3ed97e25e46a37c50a9d1a6eca5a277fb58a2e7b4.json b/backend/.sqlx/query-668edc2f84eccf5db6b8daa3ed97e25e46a37c50a9d1a6eca5a277fb58a2e7b4.json new file mode 100644 index 0000000000..7520bc8879 --- /dev/null +++ b/backend/.sqlx/query-668edc2f84eccf5db6b8daa3ed97e25e46a37c50a9d1a6eca5a277fb58a2e7b4.json @@ -0,0 +1,70 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n script_path, \n is_flow, \n workspace_id, \n edited_by, \n email, \n path, \n error_handler_path as \"error_handler_path: _\", \n error_handler_args as \"error_handler_args: _\", \n retry as \"retry: _\" \n FROM email_trigger \n WHERE local_part = $1 \n AND workspaced_local_part = FALSE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "error_handler_path: _", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "error_handler_args: _", + "type_info": "Jsonb" + }, + { + "ordinal": 8, + "name": "retry: _", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + true, + true + ] + }, + "hash": "668edc2f84eccf5db6b8daa3ed97e25e46a37c50a9d1a6eca5a277fb58a2e7b4" +} diff --git a/backend/.sqlx/query-67170c7e1a3bfeab685716ec352271f41021a4be2e351e6ef96d7af70358f0aa.json b/backend/.sqlx/query-67170c7e1a3bfeab685716ec352271f41021a4be2e351e6ef96d7af70358f0aa.json new file mode 100644 index 0000000000..a67274af63 --- /dev/null +++ b/backend/.sqlx/query-67170c7e1a3bfeab685716ec352271f41021a4be2e351e6ef96d7af70358f0aa.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + EXCLUDED.usage", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "67170c7e1a3bfeab685716ec352271f41021a4be2e351e6ef96d7af70358f0aa" +} diff --git a/backend/.sqlx/query-673564e6c4dcf30dae3d7a75c397998ea800860ede856e5fc6cd1f57d5408333.json b/backend/.sqlx/query-673564e6c4dcf30dae3d7a75c397998ea800860ede856e5fc6cd1f57d5408333.json deleted file mode 100644 index 8b9a46c4d0..0000000000 --- a/backend/.sqlx/query-673564e6c4dcf30dae3d7a75c397998ea800860ede856e5fc6cd1f57d5408333.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE \n capture_config \n SET \n last_server_ping = NULL \n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = 'gcp' AND \n server_id IS NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "673564e6c4dcf30dae3d7a75c397998ea800860ede856e5fc6cd1f57d5408333" -} diff --git a/backend/.sqlx/query-6907eb134dc5dbf118387e073897f86574c92de16252b2b1c475ab8146e5343d.json b/backend/.sqlx/query-6907eb134dc5dbf118387e073897f86574c92de16252b2b1c475ab8146e5343d.json deleted file mode 100644 index 2f14e723e4..0000000000 --- a/backend/.sqlx/query-6907eb134dc5dbf118387e073897f86574c92de16252b2b1c475ab8146e5343d.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n COALESCE(jc.result, jc.result) as \"result: sqlx::types::Json>\",\n jq.running as \"running: Option\",\n SUBSTR(rs.stream, $3) AS \"result_stream: Option\",\n CHAR_LENGTH(rs.stream) + 1 AS stream_offset\n FROM (\n SELECT $1::uuid as job_id, $2::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN v2_job_queue jq ON jq.id = base.job_id AND jq.workspace_id = base.workspace_id\n LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "result: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 1, - "name": "running: Option", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "result_stream: Option", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "stream_offset", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Int4" - ] - }, - "nullable": [ - null, - false, - null, - null - ] - }, - "hash": "6907eb134dc5dbf118387e073897f86574c92de16252b2b1c475ab8146e5343d" -} diff --git a/backend/.sqlx/query-69924462c788dbc8f31aacc7f8ae588d76bf1f25d631833ce4b194818a7d1437.json b/backend/.sqlx/query-69924462c788dbc8f31aacc7f8ae588d76bf1f25d631833ce4b194818a7d1437.json deleted file mode 100644 index f25cfd9eba..0000000000 --- a/backend/.sqlx/query-69924462c788dbc8f31aacc7f8ae588d76bf1f25d631833ce4b194818a7d1437.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT result as \"result: sqlx::types::Json>\", v2_job.tag,\n v2_job_queue.running as \"running: Option\", SUBSTR(rs.stream, $3) AS \"result_stream: Option\", CHAR_LENGTH(rs.stream) AS stream_offset\n FROM v2_job\n LEFT JOIN v2_job_queue USING (id)\n LEFT JOIN v2_job_completed USING (id)\n LEFT JOIN job_result_stream rs ON rs.job_id = $2\n WHERE v2_job.id = $2 AND v2_job.workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "result: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 1, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "running: Option", - "type_info": "Bool" - }, - { - "ordinal": 3, - "name": "result_stream: Option", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "stream_offset", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Int4" - ] - }, - "nullable": [ - true, - false, - false, - null, - null - ] - }, - "hash": "69924462c788dbc8f31aacc7f8ae588d76bf1f25d631833ce4b194818a7d1437" -} diff --git a/backend/.sqlx/query-6a0b04a34032ae0e28bbb4895ad8409185d273c022c4971329f5bce85097bc22.json b/backend/.sqlx/query-6a0b04a34032ae0e28bbb4895ad8409185d273c022c4971329f5bce85097bc22.json new file mode 100644 index 0000000000..575f738537 --- /dev/null +++ b/backend/.sqlx/query-6a0b04a34032ae0e28bbb4895ad8409185d273c022c4971329f5bce85097bc22.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM folder WHERE name = $1 AND workspace_id = $2 AND $3 = ANY(owners))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "6a0b04a34032ae0e28bbb4895ad8409185d273c022c4971329f5bce85097bc22" +} diff --git a/backend/.sqlx/query-6ad24aef02d86ea507d232ad0bbf798240d4fcdd52ce2745532ca30f736d25ca.json b/backend/.sqlx/query-6ad24aef02d86ea507d232ad0bbf798240d4fcdd52ce2745532ca30f736d25ca.json new file mode 100644 index 0000000000..bf127e8cca --- /dev/null +++ b/backend/.sqlx/query-6ad24aef02d86ea507d232ad0bbf798240d4fcdd52ce2745532ca30f736d25ca.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DROP INDEX CONCURRENTLY IF EXISTS ix_job_created_at", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "6ad24aef02d86ea507d232ad0bbf798240d4fcdd52ce2745532ca30f736d25ca" +} diff --git a/backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json b/backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json deleted file mode 100644 index 97079df64f..0000000000 --- a/backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'sqs' AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53" -} diff --git a/backend/.sqlx/query-d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45.json b/backend/.sqlx/query-6b9ea0059ad1037a77f67b70d5de13776870612e8faa8d8fa577f8e7b7309f76.json similarity index 57% rename from backend/.sqlx/query-d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45.json rename to backend/.sqlx/query-6b9ea0059ad1037a77f67b70d5de13776870612e8faa8d8fa577f8e7b7309f76.json index 3422605ef1..b2e0c68ba5 100644 --- a/backend/.sqlx/query-d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45.json +++ b/backend/.sqlx/query-6b9ea0059ad1037a77f67b70d5de13776870612e8faa8d8fa577f8e7b7309f76.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id AS \"id!\", flow_status, suspend AS \"suspend!\", script_path\n FROM v2_as_queue\n WHERE id = $1\n ", + "query": "\n SELECT j.id AS \"id!\", COALESCE(s.flow_status, s.workflow_as_code_status) as flow_status, q.suspend AS \"suspend!\", j.runnable_path as script_path\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE j.id = $1\n ", "describe": { "columns": [ { @@ -30,11 +30,11 @@ ] }, "nullable": [ - true, - true, - true, + false, + null, + false, true ] }, - "hash": "d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45" + "hash": "6b9ea0059ad1037a77f67b70d5de13776870612e8faa8d8fa577f8e7b7309f76" } diff --git a/backend/.sqlx/query-6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe.json b/backend/.sqlx/query-6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe.json new file mode 100644 index 0000000000..d7f5fc45d4 --- /dev/null +++ b/backend/.sqlx/query-6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe.json @@ -0,0 +1,62 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title)\n VALUES ($1, $2, $3, $4, $5)\n RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "flow_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "title", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "updated_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + false, + false + ] + }, + "hash": "6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe" +} diff --git a/backend/.sqlx/query-6c03fc7e623afcdb11b55390ac79d9dd236e694c0af5af0ebd90df940d893258.json b/backend/.sqlx/query-6c03fc7e623afcdb11b55390ac79d9dd236e694c0af5af0ebd90df940d893258.json new file mode 100644 index 0000000000..bfebea0664 --- /dev/null +++ b/backend/.sqlx/query-6c03fc7e623afcdb11b55390ac79d9dd236e694c0af5af0ebd90df940d893258.json @@ -0,0 +1,251 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n j.parent_job,\n j.created_by,\n cj.duration_ms,\n cj.status = 'success' OR cj.status = 'skipped' AS success,\n j.runnable_id AS script_hash,\n j.runnable_path AS script_path,\n j.args,\n cj.result,\n cj.deleted,\n cj.status = 'canceled' AS canceled,\n cj.canceled_by,\n cj.canceled_reason,\n j.kind AS job_kind,\n CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END AS schedule_path,\n j.permissioned_as,\n j.flow_step_id IS NOT NULL AS is_flow_step,\n j.script_lang AS language,\n cj.status = 'skipped' AS is_skipped,\n j.permissioned_as_email AS email,\n j.visible_to_owner,\n cj.memory_peak AS mem_peak,\n j.tag,\n cj.completed_at,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(j.created_by), 0) + \n COALESCE(LENGTH(j.runnable_path), 0) + \n COALESCE(LENGTH(j.args::text), 0) + \n COALESCE(LENGTH(cj.result::text), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_job_completed AS cj\n JOIN v2_job AS j ON cj.id = j.id\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE (cj.completed_at > $1 AND cj.completed_at < $3)\n OR cj.id = ANY($2)\n ORDER BY completed_at ASC\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT \n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY completed_at ASC \n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes,\n ROW_NUMBER() OVER (ORDER BY completed_at ASC) AS row_num\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option\",\n script_path,\n args AS \"args: sqlx::types::Json>>\",\n result AS \"result: sqlx::types::Json>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n completed_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $4 OR row_num = 1\n ORDER BY completed_at ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "success!", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "script_hash!: Option", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "args: sqlx::types::Json>>", + "type_info": "Jsonb" + }, + { + "ordinal": 9, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 10, + "name": "deleted!", + "type_info": "Bool" + }, + { + "ordinal": 11, + "name": "canceled!", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "canceled_reason", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "job_kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent" + ] + } + } + } + }, + { + "ordinal": 15, + "name": "schedule_path", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "permissioned_as!", + "type_info": "Varchar" + }, + { + "ordinal": 17, + "name": "is_flow_step!", + "type_info": "Bool" + }, + { + "ordinal": 18, + "name": "language: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + }, + { + "ordinal": 19, + "name": "is_skipped!", + "type_info": "Bool" + }, + { + "ordinal": 20, + "name": "email!", + "type_info": "Varchar" + }, + { + "ordinal": 21, + "name": "visible_to_owner!", + "type_info": "Bool" + }, + { + "ordinal": 22, + "name": "mem_peak", + "type_info": "Int4" + }, + { + "ordinal": 23, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 24, + "name": "created_at!", + "type_info": "Timestamptz" + }, + { + "ordinal": 25, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 26, + "name": "logs", + "type_info": "Text" + }, + { + "ordinal": 27, + "name": "log_offset?", + "type_info": "Int4" + }, + { + "ordinal": 28, + "name": "log_file_index", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Timestamptz", + "UuidArray", + "Timestamptz", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + false, + null, + true, + true, + true, + true, + false, + null, + true, + true, + false, + null, + false, + null, + true, + null, + false, + false, + true, + false, + false, + true, + true, + false, + true + ] + }, + "hash": "6c03fc7e623afcdb11b55390ac79d9dd236e694c0af5af0ebd90df940d893258" +} diff --git a/backend/.sqlx/query-6c962f9471b0b1fe385a93789ec46bee53a07c8d1264eeb44bc94233bc06bbfd.json b/backend/.sqlx/query-6c962f9471b0b1fe385a93789ec46bee53a07c8d1264eeb44bc94233bc06bbfd.json deleted file mode 100644 index 511fec6586..0000000000 --- a/backend/.sqlx/query-6c962f9471b0b1fe385a93789ec46bee53a07c8d1264eeb44bc94233bc06bbfd.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM dependency_map\n WHERE importer_path = $1 AND importer_kind = $3::text::IMPORTER_KIND\n AND workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "6c962f9471b0b1fe385a93789ec46bee53a07c8d1264eeb44bc94233bc06bbfd" -} diff --git a/backend/.sqlx/query-6cc922a5bbd348c938a9d1431aaa0f24f078ea814b429d44403aca1e5002e750.json b/backend/.sqlx/query-6cc922a5bbd348c938a9d1431aaa0f24f078ea814b429d44403aca1e5002e750.json new file mode 100644 index 0000000000..4b9bfe326b --- /dev/null +++ b/backend/.sqlx/query-6cc922a5bbd348c938a9d1431aaa0f24f078ea814b429d44403aca1e5002e750.json @@ -0,0 +1,216 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT \n j.id, j.workspace_id, j.runnable_id AS \"runnable_id: ScriptHash\", q.scheduled_for, q.started_at, j.parent_job, j.flow_innermost_root_job, j.runnable_path, j.kind as \"kind!: JobKind\", j.permissioned_as, \n j.created_by, j.script_lang AS \"script_lang: ScriptLang\", j.permissioned_as_email, j.flow_step_id, j.trigger_kind AS \"trigger_kind: JobTriggerKind\", j.trigger, j.priority, j.concurrent_limit, j.tag, j.cache_ttl\n FROM v2_job j LEFT JOIN v2_job_queue q ON j.id = q.id\n WHERE j.id = $1 AND j.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "runnable_id: ScriptHash", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 6, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 7, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent" + ] + } + } + } + }, + { + "ordinal": 9, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "script_lang: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + }, + { + "ordinal": 12, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "trigger_kind: JobTriggerKind", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt" + ] + } + } + } + }, + { + "ordinal": 15, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 17, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 18, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 19, + "name": "cache_ttl", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + false, + true, + true, + true, + true, + false, + false, + false, + true, + false, + true, + true, + true, + true, + true, + false, + true + ] + }, + "hash": "6cc922a5bbd348c938a9d1431aaa0f24f078ea814b429d44403aca1e5002e750" +} diff --git a/backend/.sqlx/query-6d4a6df7ec9a5200d11e5ba3884caa1e3d593f2ebf5beddcd3f9d530023ab4e5.json b/backend/.sqlx/query-6d4a6df7ec9a5200d11e5ba3884caa1e3d593f2ebf5beddcd3f9d530023ab4e5.json new file mode 100644 index 0000000000..22b585ab41 --- /dev/null +++ b/backend/.sqlx/query-6d4a6df7ec9a5200d11e5ba3884caa1e3d593f2ebf5beddcd3f9d530023ab4e5.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, versions[array_upper(versions, 1)] as version FROM flow WHERE workspace_id = $1 AND archived = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "version", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "6d4a6df7ec9a5200d11e5ba3884caa1e3d593f2ebf5beddcd3f9d530023ab4e5" +} diff --git a/backend/.sqlx/query-14abf759dae7ba5c38017ba6001927c6df0653a02b87bcea939066e39ebcf24d.json b/backend/.sqlx/query-6d7a4185063dbcca0dbea1b002330d622c9d2d844a2ab3938e6ec23c6150fb40.json similarity index 80% rename from backend/.sqlx/query-14abf759dae7ba5c38017ba6001927c6df0653a02b87bcea939066e39ebcf24d.json rename to backend/.sqlx/query-6d7a4185063dbcca0dbea1b002330d622c9d2d844a2ab3938e6ec23c6150fb40.json index 29aaf47e88..0161985a7f 100644 --- a/backend/.sqlx/query-14abf759dae7ba5c38017ba6001927c6df0653a02b87bcea939066e39ebcf24d.json +++ b/backend/.sqlx/query-6d7a4185063dbcca0dbea1b002330d622c9d2d844a2ab3938e6ec23c6150fb40.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO concurrency_locks (id, last_locked_at, owner)\n VALUES ($1, now(), $2)\n ON CONFLICT (id)\n DO UPDATE SET\n last_locked_at = now(),\n owner = $2", + "query": "INSERT INTO concurrency_locks (id, last_locked_at, owner)\n VALUES ($1, now(), $2)\n ON CONFLICT (id)\n DO UPDATE SET\n last_locked_at = now(),\n owner = EXCLUDED.owner", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "14abf759dae7ba5c38017ba6001927c6df0653a02b87bcea939066e39ebcf24d" + "hash": "6d7a4185063dbcca0dbea1b002330d622c9d2d844a2ab3938e6ec23c6150fb40" } diff --git a/backend/.sqlx/query-6e70ebf078ac04a2933d2f83791973e8fc108d9f32be8a8501391052d76e191e.json b/backend/.sqlx/query-6e70ebf078ac04a2933d2f83791973e8fc108d9f32be8a8501391052d76e191e.json deleted file mode 100644 index d95c8c9b5c..0000000000 --- a/backend/.sqlx/query-6e70ebf078ac04a2933d2f83791973e8fc108d9f32be8a8501391052d76e191e.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE kafka_trigger SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND server_id = $4 AND enabled IS TRUE RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "6e70ebf078ac04a2933d2f83791973e8fc108d9f32be8a8501391052d76e191e" -} diff --git a/backend/.sqlx/query-6ec1e81e1698c754ca8a660cf6a3ae66e75b487901befd647839dd2bc2233bf4.json b/backend/.sqlx/query-6ec1e81e1698c754ca8a660cf6a3ae66e75b487901befd647839dd2bc2233bf4.json new file mode 100644 index 0000000000..6321ca9156 --- /dev/null +++ b/backend/.sqlx/query-6ec1e81e1698c754ca8a660cf6a3ae66e75b487901befd647839dd2bc2233bf4.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE job_logs SET logs = $1 WHERE workspace_id = $2 AND job_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "6ec1e81e1698c754ca8a660cf6a3ae66e75b487901befd647839dd2bc2233bf4" +} diff --git a/backend/.sqlx/query-124e67b0cee1baa6295846db4ad6242a39dd40186f1dbb48ad3018bd9f6913ec.json b/backend/.sqlx/query-6f660e55963ac74db95c44fc95da542a18812b403641104bbd24599ee4b9d187.json similarity index 51% rename from backend/.sqlx/query-124e67b0cee1baa6295846db4ad6242a39dd40186f1dbb48ad3018bd9f6913ec.json rename to backend/.sqlx/query-6f660e55963ac74db95c44fc95da542a18812b403641104bbd24599ee4b9d187.json index 51a1ec68cb..e623c0c1e0 100644 --- a/backend/.sqlx/query-124e67b0cee1baa6295846db4ad6242a39dd40186f1dbb48ad3018bd9f6913ec.json +++ b/backend/.sqlx/query-6f660e55963ac74db95c44fc95da542a18812b403641104bbd24599ee4b9d187.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n w.premium,\n COALESCE(cw.is_past_due, false) as \"is_past_due!\",\n cw.max_tolerated_executions\n FROM\n workspace w\n LEFT JOIN cloud_workspace_settings cw ON cw.workspace_id = w.id\n WHERE\n w.id = $1\n ", + "query": "\n SELECT\n w.premium,\n COALESCE(cw.is_past_due, false) as \"is_past_due!\",\n cw.max_tolerated_executions\n FROM\n workspace w\n LEFT JOIN cloud_workspace_settings cw ON cw.workspace_id = w.id\n WHERE\n w.id = $1\n ", "describe": { "columns": [ { @@ -30,5 +30,5 @@ true ] }, - "hash": "124e67b0cee1baa6295846db4ad6242a39dd40186f1dbb48ad3018bd9f6913ec" + "hash": "6f660e55963ac74db95c44fc95da542a18812b403641104bbd24599ee4b9d187" } diff --git a/backend/.sqlx/query-70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b.json b/backend/.sqlx/query-70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b.json new file mode 100644 index 0000000000..7cfd1e5b64 --- /dev/null +++ b/backend/.sqlx/query-70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n j.id AS \"id!\", j.workspace_id AS \"workspace_id!\", j.parent_job, j.flow_step_id IS NOT NULL AS \"is_flow_step?\",\n COALESCE(s.flow_status, s.workflow_as_code_status) AS \"flow_status: Box\", r.ping AS last_ping, j.same_worker AS \"same_worker?\"\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE q.running = true AND q.suspend = 0 AND q.suspend_until IS null AND q.scheduled_for <= now()\n AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode')\n AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval\n AND q.canceled_by IS NULL\n \n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "is_flow_step?", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "flow_status: Box", + "type_info": "Jsonb" + }, + { + "ordinal": 5, + "name": "last_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "same_worker?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + true, + null, + null, + true, + false + ] + }, + "hash": "70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b" +} diff --git a/backend/.sqlx/query-70ddcf86865a315934843285e8ec618c47a5acbe400d79840c4a2d86f8886393.json b/backend/.sqlx/query-70ddcf86865a315934843285e8ec618c47a5acbe400d79840c4a2d86f8886393.json deleted file mode 100644 index 82333559eb..0000000000 --- a/backend/.sqlx/query-70ddcf86865a315934843285e8ec618c47a5acbe400d79840c4a2d86f8886393.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n c.id IS NOT NULL AS completed,\n CASE\n WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n ELSE false\n END AS running,\n CASE WHEN $7::BOOLEAN THEN NULL ELSE SUBSTR(logs, GREATEST($1 - log_offset, 0)) END AS logs,\n SUBSTR(rs.stream, $8) AS new_result_stream,\n COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json>\",\n COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json>\",\n CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset,\n CHAR_LENGTH(rs.stream) + 1 AS stream_offset,\n created_by AS \"created_by!\",\n CASE WHEN $4::BOOLEAN THEN (\n SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n ) END AS progress,\n rs.stream AS \"result_stream: Option\"\n FROM v2_job j\n LEFT JOIN v2_job_queue q USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status f USING (id)\n LEFT JOIN v2_job_completed c USING (id)\n LEFT JOIN job_result_stream rs ON rs.job_id = $3\n LEFT JOIN job_logs ON job_logs.job_id = $3\n WHERE j.workspace_id = $2 AND j.id = $3\n AND ($6::text[] IS NULL OR j.tag = ANY($6))", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "completed", - "type_info": "Bool" - }, - { - "ordinal": 1, - "name": "running", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "logs", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "new_result_stream", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "mem_peak", - "type_info": "Int4" - }, - { - "ordinal": 5, - "name": "flow_status: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 6, - "name": "workflow_as_code_status: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 7, - "name": "log_offset", - "type_info": "Int4" - }, - { - "ordinal": 8, - "name": "stream_offset", - "type_info": "Int4" - }, - { - "ordinal": 9, - "name": "created_by!", - "type_info": "Varchar" - }, - { - "ordinal": 10, - "name": "progress", - "type_info": "Int4" - }, - { - "ordinal": 11, - "name": "result_stream: Option", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Int4", - "Text", - "Uuid", - "Bool", - "Bool", - "TextArray", - "Bool", - "Int4" - ] - }, - "nullable": [ - null, - null, - null, - null, - null, - null, - null, - null, - null, - false, - null, - false - ] - }, - "hash": "70ddcf86865a315934843285e8ec618c47a5acbe400d79840c4a2d86f8886393" -} diff --git a/backend/.sqlx/query-71c945f93c0a1b561a85e8462b1687a54bd098cf6e84f57e5755eb84e1552345.json b/backend/.sqlx/query-71c945f93c0a1b561a85e8462b1687a54bd098cf6e84f57e5755eb84e1552345.json new file mode 100644 index 0000000000..da3b4cc1b9 --- /dev/null +++ b/backend/.sqlx/query-71c945f93c0a1b561a85e8462b1687a54bd098cf6e84f57e5755eb84e1552345.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT draft_only FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "draft_only", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "71c945f93c0a1b561a85e8462b1687a54bd098cf6e84f57e5755eb84e1552345" +} diff --git a/backend/.sqlx/query-72076c1c210c57ef5db26620063c750c4a7eef6dcb2f4a91c7a3ddc2d367e3ba.json b/backend/.sqlx/query-72076c1c210c57ef5db26620063c750c4a7eef6dcb2f4a91c7a3ddc2d367e3ba.json new file mode 100644 index 0000000000..b3482a1535 --- /dev/null +++ b/backend/.sqlx/query-72076c1c210c57ef5db26620063c750c4a7eef6dcb2f4a91c7a3ddc2d367e3ba.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET slack_oauth_client_id = NULL, slack_oauth_client_secret = NULL\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "72076c1c210c57ef5db26620063c750c4a7eef6dcb2f4a91c7a3ddc2d367e3ba" +} diff --git a/backend/.sqlx/query-7274e9489b18d7ab82bad1fbff89ffe41d162b482adb04b7b898a19576af2a5e.json b/backend/.sqlx/query-7274e9489b18d7ab82bad1fbff89ffe41d162b482adb04b7b898a19576af2a5e.json deleted file mode 100644 index 63363e74ff..0000000000 --- a/backend/.sqlx/query-7274e9489b18d7ab82bad1fbff89ffe41d162b482adb04b7b898a19576af2a5e.json +++ /dev/null @@ -1,251 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n cj.parent_job,\n cj.created_by,\n cj.duration_ms,\n cj.success,\n cj.script_hash,\n cj.script_path,\n cj.args,\n cj.result,\n cj.deleted,\n cj.canceled,\n cj.canceled_by,\n cj.canceled_reason,\n cj.job_kind,\n cj.schedule_path,\n cj.permissioned_as,\n cj.is_flow_step,\n cj.language,\n cj.is_skipped,\n cj.email,\n cj.visible_to_owner,\n cj.mem_peak,\n cj.tag,\n cj.created_at,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(cj.created_by), 0) + \n COALESCE(LENGTH(cj.script_path), 0) + \n COALESCE(LENGTH(cj.args::text), 0) + \n COALESCE(LENGTH(cj.result::text), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_as_completed_job AS cj\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE (cj.created_at > $1 AND cj.created_at < $3)\n OR cj.id = ANY($2)\n ORDER BY created_at ASC\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT \n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY created_at ASC \n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes,\n ROW_NUMBER() OVER (ORDER BY created_at ASC) AS row_num\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option\",\n script_path,\n args AS \"args: sqlx::types::Json>>\",\n result AS \"result: sqlx::types::Json>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n created_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $4 OR row_num = 1\n ORDER BY created_at ASC", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id!", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "workspace_id!", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "parent_job", - "type_info": "Uuid" - }, - { - "ordinal": 3, - "name": "created_by!", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "duration_ms!", - "type_info": "Int8" - }, - { - "ordinal": 5, - "name": "success!", - "type_info": "Bool" - }, - { - "ordinal": 6, - "name": "script_hash!: Option", - "type_info": "Int8" - }, - { - "ordinal": 7, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "args: sqlx::types::Json>>", - "type_info": "Jsonb" - }, - { - "ordinal": 9, - "name": "result: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 10, - "name": "deleted!", - "type_info": "Bool" - }, - { - "ordinal": 11, - "name": "canceled!", - "type_info": "Bool" - }, - { - "ordinal": 12, - "name": "canceled_by", - "type_info": "Varchar" - }, - { - "ordinal": 13, - "name": "canceled_reason", - "type_info": "Text" - }, - { - "ordinal": 14, - "name": "job_kind!: JobKind", - "type_info": { - "Custom": { - "name": "job_kind", - "kind": { - "Enum": [ - "script", - "preview", - "flow", - "dependencies", - "flowpreview", - "script_hub", - "identity", - "flowdependencies", - "http", - "graphql", - "postgresql", - "noop", - "appdependencies", - "deploymentcallback", - "singlescriptflow", - "flowscript", - "flownode", - "appscript", - "aiagent" - ] - } - } - } - }, - { - "ordinal": 15, - "name": "schedule_path", - "type_info": "Varchar" - }, - { - "ordinal": 16, - "name": "permissioned_as!", - "type_info": "Varchar" - }, - { - "ordinal": 17, - "name": "is_flow_step!", - "type_info": "Bool" - }, - { - "ordinal": 18, - "name": "language: ScriptLang", - "type_info": { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java", - "duckdb", - "ruby" - ] - } - } - } - }, - { - "ordinal": 19, - "name": "is_skipped!", - "type_info": "Bool" - }, - { - "ordinal": 20, - "name": "email!", - "type_info": "Varchar" - }, - { - "ordinal": 21, - "name": "visible_to_owner!", - "type_info": "Bool" - }, - { - "ordinal": 22, - "name": "mem_peak", - "type_info": "Int4" - }, - { - "ordinal": 23, - "name": "tag!", - "type_info": "Varchar" - }, - { - "ordinal": 24, - "name": "created_at!", - "type_info": "Timestamptz" - }, - { - "ordinal": 25, - "name": "started_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 26, - "name": "logs", - "type_info": "Text" - }, - { - "ordinal": 27, - "name": "log_offset?", - "type_info": "Int4" - }, - { - "ordinal": 28, - "name": "log_file_index", - "type_info": "TextArray" - } - ], - "parameters": { - "Left": [ - "Timestamptz", - "UuidArray", - "Timestamptz", - "Int8" - ] - }, - "nullable": [ - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - false, - true - ] - }, - "hash": "7274e9489b18d7ab82bad1fbff89ffe41d162b482adb04b7b898a19576af2a5e" -} diff --git a/backend/.sqlx/query-72956f508f66312807738399b57aa01a048fa4f9281327cf5b78111178424b43.json b/backend/.sqlx/query-72956f508f66312807738399b57aa01a048fa4f9281327cf5b78111178424b43.json deleted file mode 100644 index 2ffdf141b1..0000000000 --- a/backend/.sqlx/query-72956f508f66312807738399b57aa01a048fa4f9281327cf5b78111178424b43.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n id AS \"id!\", workspace_id AS \"workspace_id!\", parent_job, is_flow_step,\n flow_status AS \"flow_status: Box\", last_ping, same_worker\n FROM v2_as_queue\n WHERE running = true AND suspend = 0 AND suspend_until IS null AND scheduled_for <= now()\n AND (job_kind = 'flow' OR job_kind = 'flowpreview' OR job_kind = 'flownode')\n AND last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval\n AND canceled = false\n \n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id!", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "workspace_id!", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "parent_job", - "type_info": "Uuid" - }, - { - "ordinal": 3, - "name": "is_flow_step", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "flow_status: Box", - "type_info": "Jsonb" - }, - { - "ordinal": 5, - "name": "last_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 6, - "name": "same_worker", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - true, - true, - true, - true, - true, - true, - true - ] - }, - "hash": "72956f508f66312807738399b57aa01a048fa4f9281327cf5b78111178424b43" -} diff --git a/backend/.sqlx/query-5430f7728c1e9b539cc8aad29ca9e6733943278998d3df62a9486607827e59ec.json b/backend/.sqlx/query-72d3ebb05ac1ffeb0e8d0a3146d95bb5b90e7c4d1dc2c8a6ef06eddf6678f230.json similarity index 79% rename from backend/.sqlx/query-5430f7728c1e9b539cc8aad29ca9e6733943278998d3df62a9486607827e59ec.json rename to backend/.sqlx/query-72d3ebb05ac1ffeb0e8d0a3146d95bb5b90e7c4d1dc2c8a6ef06eddf6678f230.json index f3961847f4..0a78509ca3 100644 --- a/backend/.sqlx/query-5430f7728c1e9b539cc8aad29ca9e6733943278998d3df62a9486607827e59ec.json +++ b/backend/.sqlx/query-72d3ebb05ac1ffeb0e8d0a3146d95bb5b90e7c4d1dc2c8a6ef06eddf6678f230.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC, devops DESC\n LIMIT $1 OFFSET $2", + "query": "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC, devops DESC\n LIMIT $1 OFFSET $2", "describe": { "columns": [ { @@ -47,6 +47,11 @@ "ordinal": 8, "name": "username", "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "first_time_user", + "type_info": "Bool" } ], "parameters": { @@ -64,8 +69,9 @@ false, true, true, - true + true, + false ] }, - "hash": "5430f7728c1e9b539cc8aad29ca9e6733943278998d3df62a9486607827e59ec" + "hash": "72d3ebb05ac1ffeb0e8d0a3146d95bb5b90e7c4d1dc2c8a6ef06eddf6678f230" } diff --git a/backend/.sqlx/query-73c1c88bdf26ea0559b83314fed7a67d850e4e4dd60f4424ffb0b6f472acc8d5.json b/backend/.sqlx/query-73c1c88bdf26ea0559b83314fed7a67d850e4e4dd60f4424ffb0b6f472acc8d5.json new file mode 100644 index 0000000000..cf36aed51a --- /dev/null +++ b/backend/.sqlx/query-73c1c88bdf26ea0559b83314fed7a67d850e4e4dd60f4424ffb0b6f472acc8d5.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value->'instance_catalog_db_status' FROM global_settings WHERE name = 'ducklake_settings'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "73c1c88bdf26ea0559b83314fed7a67d850e4e4dd60f4424ffb0b6f472acc8d5" +} diff --git a/backend/.sqlx/query-2ae44acae7ac80e191b37071baa26e57488489177fe596940d3698d843a93666.json b/backend/.sqlx/query-73fcf81d272c1613e094d60c0a088f9d694bc37caacef7269e3738de8b5f6013.json similarity index 56% rename from backend/.sqlx/query-2ae44acae7ac80e191b37071baa26e57488489177fe596940d3698d843a93666.json rename to backend/.sqlx/query-73fcf81d272c1613e094d60c0a088f9d694bc37caacef7269e3738de8b5f6013.json index 405902863c..00f7ec50d2 100644 --- a/backend/.sqlx/query-2ae44acae7ac80e191b37071baa26e57488489177fe596940d3698d843a93666.json +++ b/backend/.sqlx/query-73fcf81d272c1613e094d60c0a088f9d694bc37caacef7269e3738de8b5f6013.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT script_path FROM v2_as_queue WHERE id = $1", + "query": "SELECT key FROM debounce_key WHERE job_id = $1", "describe": { "columns": [ { "ordinal": 0, - "name": "script_path", + "name": "key", "type_info": "Varchar" } ], @@ -15,8 +15,8 @@ ] }, "nullable": [ - true + false ] }, - "hash": "2ae44acae7ac80e191b37071baa26e57488489177fe596940d3698d843a93666" + "hash": "73fcf81d272c1613e094d60c0a088f9d694bc37caacef7269e3738de8b5f6013" } diff --git a/backend/.sqlx/query-26829b40c9cbdc466154dbb9cea3c2a6a1378d87c0e2f0b5c9cda882b52e3eb0.json b/backend/.sqlx/query-748b63f877498d6d279fdf23894611e11c7a05d69b4b7907f8909aec82fb1eb2.json similarity index 67% rename from backend/.sqlx/query-26829b40c9cbdc466154dbb9cea3c2a6a1378d87c0e2f0b5c9cda882b52e3eb0.json rename to backend/.sqlx/query-748b63f877498d6d279fdf23894611e11c7a05d69b4b7907f8909aec82fb1eb2.json index c2639ca086..d710d59055 100644 --- a/backend/.sqlx/query-26829b40c9cbdc466154dbb9cea3c2a6a1378d87c0e2f0b5c9cda882b52e3eb0.json +++ b/backend/.sqlx/query-748b63f877498d6d279fdf23894611e11c7a05d69b4b7907f8909aec82fb1eb2.json @@ -1,16 +1,15 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, edited_at, created_by)\n SELECT $2, path, value, description, resource_type, extra_perms, edited_at, $3\n FROM resource \n WHERE workspace_id = $1", + "query": "INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, edited_at, created_by)\n SELECT $2, path, value, description, resource_type, extra_perms, edited_at, created_by\n FROM resource\n WHERE workspace_id = $1", "describe": { "columns": [], "parameters": { "Left": [ "Text", - "Varchar", "Varchar" ] }, "nullable": [] }, - "hash": "26829b40c9cbdc466154dbb9cea3c2a6a1378d87c0e2f0b5c9cda882b52e3eb0" + "hash": "748b63f877498d6d279fdf23894611e11c7a05d69b4b7907f8909aec82fb1eb2" } diff --git a/backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json b/backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json deleted file mode 100644 index 0f72e41b9f..0000000000 --- a/backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'postgres' AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c" -} diff --git a/backend/.sqlx/query-681be1486a7853c42b1a74aa523a6a1cd42a79852952115e6f30db0dc6ee4b6e.json b/backend/.sqlx/query-755a9c2f19d3befe68ebffca43abae28b5ba639731cb4867ab2a8e0acdcc9c32.json similarity index 55% rename from backend/.sqlx/query-681be1486a7853c42b1a74aa523a6a1cd42a79852952115e6f30db0dc6ee4b6e.json rename to backend/.sqlx/query-755a9c2f19d3befe68ebffca43abae28b5ba639731cb4867ab2a8e0acdcc9c32.json index 01693cc7c5..112b8fe253 100644 --- a/backend/.sqlx/query-681be1486a7853c42b1a74aa523a6a1cd42a79852952115e6f30db0dc6ee4b6e.json +++ b/backend/.sqlx/query-755a9c2f19d3befe68ebffca43abae28b5ba639731cb4867ab2a8e0acdcc9c32.json @@ -1,16 +1,15 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO flow (\n workspace_id, path, summary, description, value, edited_by, edited_at,\n archived, schema, extra_perms, dependency_job, draft_only, tag,\n ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only,\n concurrency_key, versions, on_behalf_of_email, lock_error_logs\n )\n SELECT $2, path, summary, description, value, $3, edited_at,\n archived, schema, extra_perms, NULL, draft_only, tag,\n ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only,\n concurrency_key, ARRAY[]::bigint[], on_behalf_of_email, lock_error_logs\n FROM flow \n WHERE workspace_id = $1", + "query": "INSERT INTO flow (\n workspace_id, path, summary, description, value, edited_by, edited_at,\n archived, schema, extra_perms, dependency_job, draft_only, tag,\n ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only,\n concurrency_key, versions, on_behalf_of_email, lock_error_logs\n )\n SELECT $2, path, summary, description, value, edited_by, edited_at,\n archived, schema, extra_perms, NULL, draft_only, tag,\n ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only,\n concurrency_key, ARRAY[]::bigint[], on_behalf_of_email, lock_error_logs\n FROM flow\n WHERE workspace_id = $1", "describe": { "columns": [], "parameters": { "Left": [ "Text", - "Varchar", "Varchar" ] }, "nullable": [] }, - "hash": "681be1486a7853c42b1a74aa523a6a1cd42a79852952115e6f30db0dc6ee4b6e" + "hash": "755a9c2f19d3befe68ebffca43abae28b5ba639731cb4867ab2a8e0acdcc9c32" } diff --git a/backend/.sqlx/query-7580917b8c791207556e2ed6734edb07863c362cf2a8c4624a1fc6ae3136a568.json b/backend/.sqlx/query-7580917b8c791207556e2ed6734edb07863c362cf2a8c4624a1fc6ae3136a568.json new file mode 100644 index 0000000000..5ea14fd63f --- /dev/null +++ b/backend/.sqlx/query-7580917b8c791207556e2ed6734edb07863c362cf2a8c4624a1fc6ae3136a568.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + EXCLUDED.usage", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "7580917b8c791207556e2ed6734edb07863c362cf2a8c4624a1fc6ae3136a568" +} diff --git a/backend/.sqlx/query-76774e6f72c8c8b7473487e4176dc17b17372b7292e39d3888a93ff4fe49e4f5.json b/backend/.sqlx/query-76774e6f72c8c8b7473487e4176dc17b17372b7292e39d3888a93ff4fe49e4f5.json new file mode 100644 index 0000000000..694ed1887f --- /dev/null +++ b/backend/.sqlx/query-76774e6f72c8c8b7473487e4176dc17b17372b7292e39d3888a93ff4fe49e4f5.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM debounce_key WHERE key = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "76774e6f72c8c8b7473487e4176dc17b17372b7292e39d3888a93ff4fe49e4f5" +} diff --git a/backend/.sqlx/query-7cd070aaca3b4f95bb6e669ac07f288bd1cc8a85625255d86e45f63e8267d34c.json b/backend/.sqlx/query-7927b80ce75d99b2a30f6b29196af000578a3c166509f032d14452cc637d884f.json similarity index 94% rename from backend/.sqlx/query-7cd070aaca3b4f95bb6e669ac07f288bd1cc8a85625255d86e45f63e8267d34c.json rename to backend/.sqlx/query-7927b80ce75d99b2a30f6b29196af000578a3c166509f032d14452cc637d884f.json index 99095e3fcc..84585a66a9 100644 --- a/backend/.sqlx/query-7cd070aaca3b4f95bb6e669ac07f288bd1cc8a85625255d86e45f63e8267d34c.json +++ b/backend/.sqlx/query-7927b80ce75d99b2a30f6b29196af000578a3c166509f032d14452cc637d884f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE schedule SET\n enabled = $1,\n email = $2\n WHERE path = $3 AND workspace_id = $4\n RETURNING\n workspace_id,\n path,\n edited_by,\n edited_at,\n schedule,\n timezone,\n enabled,\n script_path,\n is_flow,\n args AS \"args: _\",\n extra_perms,\n email,\n error,\n on_failure,\n on_failure_times,\n on_failure_exact,\n on_failure_extra_args AS \"on_failure_extra_args: _\",\n on_recovery,\n on_recovery_times,\n on_recovery_extra_args AS \"on_recovery_extra_args: _\",\n on_success,\n on_success_extra_args AS \"on_success_extra_args: _\",\n ws_error_handler_muted,\n retry,\n no_flow_overlap,\n summary,\n description,\n tag,\n paused_until,\n cron_version\n ", + "query": "\n UPDATE schedule SET\n enabled = $1,\n email = $2\n WHERE path = $3 AND workspace_id = $4\n RETURNING\n workspace_id,\n path,\n edited_by,\n edited_at,\n schedule,\n timezone,\n enabled,\n script_path,\n is_flow,\n args AS \"args: _\",\n extra_perms,\n email,\n error,\n on_failure,\n on_failure_times,\n on_failure_exact,\n on_failure_extra_args AS \"on_failure_extra_args: _\",\n on_recovery,\n on_recovery_times,\n on_recovery_extra_args AS \"on_recovery_extra_args: _\",\n on_success,\n on_success_extra_args AS \"on_success_extra_args: _\",\n ws_error_handler_muted,\n retry,\n no_flow_overlap,\n summary,\n description,\n tag,\n paused_until,\n cron_version,\n dynamic_skip\n ", "describe": { "columns": [ { @@ -152,6 +152,11 @@ "ordinal": 29, "name": "cron_version", "type_info": "Text" + }, + { + "ordinal": 30, + "name": "dynamic_skip", + "type_info": "Varchar" } ], "parameters": { @@ -192,8 +197,9 @@ true, true, true, + true, true ] }, - "hash": "7cd070aaca3b4f95bb6e669ac07f288bd1cc8a85625255d86e45f63e8267d34c" + "hash": "7927b80ce75d99b2a30f6b29196af000578a3c166509f032d14452cc637d884f" } diff --git a/backend/.sqlx/query-d6f62e25faf271876874fc09ee460313159bb6ad91227f5dec37cd28006e2add.json b/backend/.sqlx/query-79d6b757c9556cfcf0c98f52035b5f1a9036b6005764b79c415373a5d39c3211.json similarity index 54% rename from backend/.sqlx/query-d6f62e25faf271876874fc09ee460313159bb6ad91227f5dec37cd28006e2add.json rename to backend/.sqlx/query-79d6b757c9556cfcf0c98f52035b5f1a9036b6005764b79c415373a5d39c3211.json index c4c138a886..7b0d64c691 100644 --- a/backend/.sqlx/query-d6f62e25faf271876874fc09ee460313159bb6ad91227f5dec37cd28006e2add.json +++ b/backend/.sqlx/query-79d6b757c9556cfcf0c98f52035b5f1a9036b6005764b79c415373a5d39c3211.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n result AS \"result: sqlx::types::Json>\",\n result_columns,\n status = 'success' AS \"success!\"\n FROM v2_job_completed\n WHERE id = $1 AND workspace_id = $2", + "query": "\n SELECT\n result AS \"result: sqlx::types::Json>\",\n result_columns,\n status = 'success' AS \"success!\"\n FROM \n v2_job_completed\n WHERE \n id = $1 AND \n workspace_id = $2\n ", "describe": { "columns": [ { @@ -31,5 +31,5 @@ null ] }, - "hash": "d6f62e25faf271876874fc09ee460313159bb6ad91227f5dec37cd28006e2add" + "hash": "79d6b757c9556cfcf0c98f52035b5f1a9036b6005764b79c415373a5d39c3211" } diff --git a/backend/.sqlx/query-841aaef7303e5994d97b529d256e7980b81861af63ba116630603087196ce02b.json b/backend/.sqlx/query-7abd579d3ec97853ac36cc8dad29013eb133a28cd848bf8fdf9571b2ee402a3e.json similarity index 56% rename from backend/.sqlx/query-841aaef7303e5994d97b529d256e7980b81861af63ba116630603087196ce02b.json rename to backend/.sqlx/query-7abd579d3ec97853ac36cc8dad29013eb133a28cd848bf8fdf9571b2ee402a3e.json index b95af1ddc8..7d45d000ab 100644 --- a/backend/.sqlx/query-841aaef7303e5994d97b529d256e7980b81861af63ba116630603087196ce02b.json +++ b/backend/.sqlx/query-7abd579d3ec97853ac36cc8dad29013eb133a28cd848bf8fdf9571b2ee402a3e.json @@ -1,16 +1,15 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension)\n SELECT $2, name, schema, description, edited_at, $3, format_extension\n FROM resource_type \n WHERE workspace_id = $1", + "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension)\n SELECT $2, name, schema, description, edited_at, created_by, format_extension\n FROM resource_type\n WHERE workspace_id = $1", "describe": { "columns": [], "parameters": { "Left": [ "Text", - "Varchar", "Varchar" ] }, "nullable": [] }, - "hash": "841aaef7303e5994d97b529d256e7980b81861af63ba116630603087196ce02b" + "hash": "7abd579d3ec97853ac36cc8dad29013eb133a28cd848bf8fdf9571b2ee402a3e" } diff --git a/backend/.sqlx/query-7b48820a08fbb3bee0fb2dc802a0b4f28ed5507797165e8c577ce6a95d11694c.json b/backend/.sqlx/query-7b48820a08fbb3bee0fb2dc802a0b4f28ed5507797165e8c577ce6a95d11694c.json deleted file mode 100644 index 56481d8eb0..0000000000 --- a/backend/.sqlx/query-7b48820a08fbb3bee0fb2dc802a0b4f28ed5507797165e8c577ce6a95d11694c.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "7b48820a08fbb3bee0fb2dc802a0b4f28ed5507797165e8c577ce6a95d11694c" -} diff --git a/backend/.sqlx/query-7b524ee24bb78e494a93c5ea205259bd989a9d769a6a300d37ab116960438882.json b/backend/.sqlx/query-7b524ee24bb78e494a93c5ea205259bd989a9d769a6a300d37ab116960438882.json new file mode 100644 index 0000000000..59f972f50d --- /dev/null +++ b/backend/.sqlx/query-7b524ee24bb78e494a93c5ea205259bd989a9d769a6a300d37ab116960438882.json @@ -0,0 +1,101 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH result_stream AS (\n SELECT \n string_agg(stream, '' order by idx asc) as stream, \n job_id, \n max(idx) + 1 as offset \n FROM job_result_stream_v2\n WHERE job_id = $3 AND idx >= $8\n GROUP BY job_id\n )\n SELECT\n c.id IS NOT NULL AS completed,\n CASE\n WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n ELSE false\n END AS running,\n CASE WHEN $7::BOOLEAN THEN NULL ELSE SUBSTR(logs, GREATEST($1 - log_offset, 0)) END AS logs,\n rs.stream AS new_result_stream,\n COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json>\",\n (COALESCE(c.flow_status, f.flow_status)->>'stream_job')::uuid AS stream_job,\n COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json>\",\n CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset,\n rs.offset AS stream_offset,\n created_by AS \"created_by!\",\n CASE WHEN $4::BOOLEAN THEN (\n SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n ) END AS progress,\n rs.stream AS \"result_stream: Option\"\n FROM v2_job j\n LEFT JOIN v2_job_queue q USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status f USING (id)\n LEFT JOIN v2_job_completed c USING (id)\n LEFT JOIN result_stream rs ON rs.job_id = $3\n LEFT JOIN job_logs ON job_logs.job_id = $3\n WHERE j.workspace_id = $2 AND j.id = $3\n AND ($6::text[] IS NULL OR j.tag = ANY($6))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "completed", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "running", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "logs", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "new_result_stream", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "mem_peak", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "flow_status: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 6, + "name": "stream_job", + "type_info": "Uuid" + }, + { + "ordinal": 7, + "name": "workflow_as_code_status: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 8, + "name": "log_offset", + "type_info": "Int4" + }, + { + "ordinal": 9, + "name": "stream_offset", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "progress", + "type_info": "Int4" + }, + { + "ordinal": 12, + "name": "result_stream: Option", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int4", + "Text", + "Uuid", + "Bool", + "Bool", + "TextArray", + "Bool", + "Int4" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + false, + null, + null + ] + }, + "hash": "7b524ee24bb78e494a93c5ea205259bd989a9d769a6a300d37ab116960438882" +} diff --git a/backend/.sqlx/query-ca15fe5d43f0e94f50408efe5c9e359770b759e8661687b4503c4b692ecd245e.json b/backend/.sqlx/query-7bfb3b210d23f2c00a1d6a653e9df5d7df9acf74de6dcc566924de02f1807af2.json similarity index 52% rename from backend/.sqlx/query-ca15fe5d43f0e94f50408efe5c9e359770b759e8661687b4503c4b692ecd245e.json rename to backend/.sqlx/query-7bfb3b210d23f2c00a1d6a653e9df5d7df9acf74de6dcc566924de02f1807af2.json index 6812bfdfc7..19ec28bbe3 100644 --- a/backend/.sqlx/query-ca15fe5d43f0e94f50408efe5c9e359770b759e8661687b4503c4b692ecd245e.json +++ b/backend/.sqlx/query-7bfb3b210d23f2c00a1d6a653e9df5d7df9acf74de6dcc566924de02f1807af2.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT versions[array_upper(versions, 1)] FROM app WHERE path = $1 AND workspace_id = $2", + "query": "SELECT id FROM flow_version WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", "describe": { "columns": [ { "ordinal": 0, - "name": "versions", + "name": "id", "type_info": "Int8" } ], @@ -16,8 +16,8 @@ ] }, "nullable": [ - null + false ] }, - "hash": "ca15fe5d43f0e94f50408efe5c9e359770b759e8661687b4503c4b692ecd245e" + "hash": "7bfb3b210d23f2c00a1d6a653e9df5d7df9acf74de6dcc566924de02f1807af2" } diff --git a/backend/.sqlx/query-d970a0b07a2b5840d1feb1baacb834dbaf91c633d3e7e1e29c8eb7eedc53e888.json b/backend/.sqlx/query-7dc75cb67922e31ffe0f88b03a8a5bc14039aff6f023d79139b20519a9cdbe7d.json similarity index 74% rename from backend/.sqlx/query-d970a0b07a2b5840d1feb1baacb834dbaf91c633d3e7e1e29c8eb7eedc53e888.json rename to backend/.sqlx/query-7dc75cb67922e31ffe0f88b03a8a5bc14039aff6f023d79139b20519a9cdbe7d.json index c6324a22d5..33e330c061 100644 --- a/backend/.sqlx/query-d970a0b07a2b5840d1feb1baacb834dbaf91c633d3e7e1e29c8eb7eedc53e888.json +++ b/backend/.sqlx/query-7dc75cb67922e31ffe0f88b03a8a5bc14039aff6f023d79139b20519a9cdbe7d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO workspace_invite\n (workspace_id, email, is_admin, operator)\n VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, email)\n DO UPDATE SET is_admin = $3, operator = $4", + "query": "INSERT INTO workspace_invite\n (workspace_id, email, is_admin, operator)\n VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, email)\n DO UPDATE SET is_admin = EXCLUDED.is_admin, operator = EXCLUDED.operator", "describe": { "columns": [], "parameters": { @@ -13,5 +13,5 @@ }, "nullable": [] }, - "hash": "d970a0b07a2b5840d1feb1baacb834dbaf91c633d3e7e1e29c8eb7eedc53e888" + "hash": "7dc75cb67922e31ffe0f88b03a8a5bc14039aff6f023d79139b20519a9cdbe7d" } diff --git a/backend/.sqlx/query-43b376a2eff086a32cd76e54361ce3631feee1565935d2a6ddbecc17950758d1.json b/backend/.sqlx/query-7dcc77eb6da5863f7a25ab6ad83d270e5e8a52540da1726aa875fcbe2517f16a.json similarity index 67% rename from backend/.sqlx/query-43b376a2eff086a32cd76e54361ce3631feee1565935d2a6ddbecc17950758d1.json rename to backend/.sqlx/query-7dcc77eb6da5863f7a25ab6ad83d270e5e8a52540da1726aa875fcbe2517f16a.json index 5b9eeaa790..d7b8988edf 100644 --- a/backend/.sqlx/query-43b376a2eff086a32cd76e54361ce3631feee1565935d2a6ddbecc17950758d1.json +++ b/backend/.sqlx/query-7dcc77eb6da5863f7a25ab6ad83d270e5e8a52540da1726aa875fcbe2517f16a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO workspace_settings\n (workspace_id, slack_team_id, slack_name, slack_email)\n VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id) DO UPDATE SET slack_team_id = $2, slack_name = $3, slack_email = $4", + "query": "INSERT INTO workspace_settings\n (workspace_id, slack_team_id, slack_name, slack_email)\n VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id) DO UPDATE SET slack_team_id = EXCLUDED.slack_team_id, slack_name = EXCLUDED.slack_name, slack_email = EXCLUDED.slack_email", "describe": { "columns": [], "parameters": { @@ -13,5 +13,5 @@ }, "nullable": [] }, - "hash": "43b376a2eff086a32cd76e54361ce3631feee1565935d2a6ddbecc17950758d1" + "hash": "7dcc77eb6da5863f7a25ab6ad83d270e5e8a52540da1726aa875fcbe2517f16a" } diff --git a/backend/.sqlx/query-b8c66d905a6c7ffa6441c84b14ea897040069dac7367895813cc2d64a9867193.json b/backend/.sqlx/query-7dcf840fc5b329f4a591a51e24c9dacb12266f606b5ce7e8ed6110f3f381f945.json similarity index 75% rename from backend/.sqlx/query-b8c66d905a6c7ffa6441c84b14ea897040069dac7367895813cc2d64a9867193.json rename to backend/.sqlx/query-7dcf840fc5b329f4a591a51e24c9dacb12266f606b5ce7e8ed6110f3f381f945.json index 265a3c96a4..5084c21af8 100644 --- a/backend/.sqlx/query-b8c66d905a6c7ffa6441c84b14ea897040069dac7367895813cc2d64a9867193.json +++ b/backend/.sqlx/query-7dcf840fc5b329f4a591a51e24c9dacb12266f606b5ce7e8ed6110f3f381f945.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO workspace_env (workspace_id, name, value) VALUES ($1, $2, $3) ON CONFLICT (workspace_id, name) DO UPDATE SET value = $3", + "query": "INSERT INTO workspace_env (workspace_id, name, value) VALUES ($1, $2, $3) ON CONFLICT (workspace_id, name) DO UPDATE SET value = EXCLUDED.value", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "b8c66d905a6c7ffa6441c84b14ea897040069dac7367895813cc2d64a9867193" + "hash": "7dcf840fc5b329f4a591a51e24c9dacb12266f606b5ce7e8ed6110f3f381f945" } diff --git a/backend/.sqlx/query-7e31c47e02492b74cbb5702dfc5ef9f4c8faa986843024a2cd58e212f706aae8.json b/backend/.sqlx/query-7e31c47e02492b74cbb5702dfc5ef9f4c8faa986843024a2cd58e212f706aae8.json deleted file mode 100644 index eea1cfa434..0000000000 --- a/backend/.sqlx/query-7e31c47e02492b74cbb5702dfc5ef9f4c8faa986843024a2cd58e212f706aae8.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'sqs'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "7e31c47e02492b74cbb5702dfc5ef9f4c8faa986843024a2cd58e212f706aae8" -} diff --git a/backend/.sqlx/query-7ec724b84479c2f737637e91b8cbed6cae29f361167deee879b8b683ad1bf684.json b/backend/.sqlx/query-7ec724b84479c2f737637e91b8cbed6cae29f361167deee879b8b683ad1bf684.json new file mode 100644 index 0000000000..a979af6114 --- /dev/null +++ b/backend/.sqlx/query-7ec724b84479c2f737637e91b8cbed6cae29f361167deee879b8b683ad1bf684.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO debounce_stale_data (job_id, to_relock)\n VALUES ($1, $2)\n ON CONFLICT (job_id)\n DO UPDATE SET to_relock = (\n SELECT array_agg(DISTINCT x)\n FROM unnest(\n -- Combine existing array with new values, removing duplicates\n array_cat(debounce_stale_data.to_relock, EXCLUDED.to_relock)\n ) AS x\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "7ec724b84479c2f737637e91b8cbed6cae29f361167deee879b8b683ad1bf684" +} diff --git a/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json b/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json deleted file mode 100644 index 6f08d98113..0000000000 --- a/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO script (summary, description, dedicated_worker, content, workspace_id, path, hash, language, tag, created_by, lock) VALUES ('', '', true, $1, $2, $3, $4, $5, $6, $7, '') ON CONFLICT (workspace_id, hash) DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Varchar", - "Varchar", - "Int8", - { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java", - "duckdb", - "ruby" - ] - } - } - }, - "Varchar", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf" -} diff --git a/backend/.sqlx/query-805d633de90fee335f1726284eda0dbc200d45960fb8dea867492c8c7dd096d5.json b/backend/.sqlx/query-805d633de90fee335f1726284eda0dbc200d45960fb8dea867492c8c7dd096d5.json index 5500f83561..c18d338929 100644 --- a/backend/.sqlx/query-805d633de90fee335f1726284eda0dbc200d45960fb8dea867492c8c7dd096d5.json +++ b/backend/.sqlx/query-805d633de90fee335f1726284eda0dbc200d45960fb8dea867492c8c7dd096d5.json @@ -35,7 +35,7 @@ "noop", "appdependencies", "deploymentcallback", - "singlescriptflow", + "singlestepflow", "flowscript", "flownode", "appscript", diff --git a/backend/.sqlx/query-807c920bff25f56b10e88900d879cf5e8484c147e457044d6b075323b163ebaa.json b/backend/.sqlx/query-807c920bff25f56b10e88900d879cf5e8484c147e457044d6b075323b163ebaa.json deleted file mode 100644 index 1d56449d18..0000000000 --- a/backend/.sqlx/query-807c920bff25f56b10e88900d879cf5e8484c147e457044d6b075323b163ebaa.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "VACUUM v2_job, v2_job_completed, job_result_stream, job_stats, job_logs, concurrency_key, log_file, metrics", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "807c920bff25f56b10e88900d879cf5e8484c147e457044d6b075323b163ebaa" -} diff --git a/backend/.sqlx/query-8167af4650c99dc66de483d72a675a7b57946b430ad0839685797a10ac1adfd8.json b/backend/.sqlx/query-8167af4650c99dc66de483d72a675a7b57946b430ad0839685797a10ac1adfd8.json new file mode 100644 index 0000000000..0d8d788602 --- /dev/null +++ b/backend/.sqlx/query-8167af4650c99dc66de483d72a675a7b57946b430ad0839685797a10ac1adfd8.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, started_at FROM v2_job_queue WHERE id = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "started_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "8167af4650c99dc66de483d72a675a7b57946b430ad0839685797a10ac1adfd8" +} diff --git a/backend/.sqlx/query-81e997576319ae1d6d9a91f76465f6fa53892ff223f3b9d9ad3f4a1d2e720cc8.json b/backend/.sqlx/query-81e997576319ae1d6d9a91f76465f6fa53892ff223f3b9d9ad3f4a1d2e720cc8.json new file mode 100644 index 0000000000..a067ffee64 --- /dev/null +++ b/backend/.sqlx/query-81e997576319ae1d6d9a91f76465f6fa53892ff223f3b9d9ad3f4a1d2e720cc8.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM flow_conversation WHERE id = $1 AND workspace_id = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "81e997576319ae1d6d9a91f76465f6fa53892ff223f3b9d9ad3f4a1d2e720cc8" +} diff --git a/backend/.sqlx/query-83232f2db5eb1b6fef744998e60420ef920d472286cf4c1f78452446a4bcb604.json b/backend/.sqlx/query-83232f2db5eb1b6fef744998e60420ef920d472286cf4c1f78452446a4bcb604.json deleted file mode 100644 index 27a5df6de9..0000000000 --- a/backend/.sqlx/query-83232f2db5eb1b6fef744998e60420ef920d472286cf4c1f78452446a4bcb604.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO app_version\n (app_id, value, created_by, raw_app)\n SELECT app_id, value, created_by, raw_app\n FROM app_version WHERE id = $1\n RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "83232f2db5eb1b6fef744998e60420ef920d472286cf4c1f78452446a4bcb604" -} diff --git a/backend/.sqlx/query-83f64dd93b1ddc03b84681d65d9be69959987cbac1d83b64225fd1bf9ab047c9.json b/backend/.sqlx/query-83f64dd93b1ddc03b84681d65d9be69959987cbac1d83b64225fd1bf9ab047c9.json deleted file mode 100644 index 3ca69e27dd..0000000000 --- a/backend/.sqlx/query-83f64dd93b1ddc03b84681d65d9be69959987cbac1d83b64225fd1bf9ab047c9.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 \n RETURNING usage.usage", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "usage", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar" - ] - }, - "nullable": [ - false - ] - }, - "hash": "83f64dd93b1ddc03b84681d65d9be69959987cbac1d83b64225fd1bf9ab047c9" -} diff --git a/backend/.sqlx/query-85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1.json b/backend/.sqlx/query-85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1.json deleted file mode 100644 index bce7324fb6..0000000000 --- a/backend/.sqlx/query-85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag) SELECT unnest($1::uuid[]), $2, now(), $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray", - "Varchar", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1" -} diff --git a/backend/.sqlx/query-85d945cf5ade707291a161078ff96ddb29140dd72ce5115657418eec503b205d.json b/backend/.sqlx/query-85d945cf5ade707291a161078ff96ddb29140dd72ce5115657418eec503b205d.json deleted file mode 100644 index fae73f363c..0000000000 --- a/backend/.sqlx/query-85d945cf5ade707291a161078ff96ddb29140dd72ce5115657418eec503b205d.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n http_trigger \n SET \n route_path = $1, \n route_path_key = $2,\n workspaced_route = $3,\n wrap_body = $4,\n raw_string = $5,\n authentication_resource_path = $6,\n script_path = $7, \n path = $8, \n is_flow = $9, \n http_method = $10, \n static_asset_config = $11, \n edited_by = $12, \n email = $13, \n is_async = $14, \n authentication_method = $15, \n summary = $16,\n description = $17,\n edited_at = now(), \n is_static_website = $18,\n error_handler_path = $19,\n error_handler_args = $20,\n retry = $21\n WHERE \n workspace_id = $22 AND \n path = $23\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Bool", - "Bool", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Bool", - { - "Custom": { - "name": "http_method", - "kind": { - "Enum": [ - "get", - "post", - "put", - "delete", - "patch" - ] - } - } - }, - "Jsonb", - "Varchar", - "Varchar", - "Bool", - { - "Custom": { - "name": "authentication_method", - "kind": { - "Enum": [ - "none", - "windmill", - "api_key", - "basic_http", - "custom_script", - "signature" - ] - } - } - }, - "Varchar", - "Text", - "Bool", - "Varchar", - "Jsonb", - "Jsonb", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "85d945cf5ade707291a161078ff96ddb29140dd72ce5115657418eec503b205d" -} diff --git a/backend/.sqlx/query-85ec6c6384ab77df102d05caf82079e65a5300a52a292b9143b755087b9f9c4c.json b/backend/.sqlx/query-85ec6c6384ab77df102d05caf82079e65a5300a52a292b9143b755087b9f9c4c.json new file mode 100644 index 0000000000..5573e5208b --- /dev/null +++ b/backend/.sqlx/query-85ec6c6384ab77df102d05caf82079e65a5300a52a292b9143b755087b9f9c4c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM resource WHERE path = $1 AND workspace_id = $2 RETURNING path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "85ec6c6384ab77df102d05caf82079e65a5300a52a292b9143b755087b9f9c4c" +} diff --git a/backend/.sqlx/query-89940a53f29b173b6a8717f057a81a4f2edad152581950fdd80d758a0d242c17.json b/backend/.sqlx/query-89940a53f29b173b6a8717f057a81a4f2edad152581950fdd80d758a0d242c17.json deleted file mode 100644 index 25dd18003c..0000000000 --- a/backend/.sqlx/query-89940a53f29b173b6a8717f057a81a4f2edad152581950fdd80d758a0d242c17.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT usr.email, usage.executions\n FROM usr\n , LATERAL (\n SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_as_completed_job\n WHERE workspace_id = $1\n AND job_kind NOT IN ('flow', 'flowpreview', 'flownode')\n AND email = usr.email\n AND now() - '1 week'::interval < created_at\n ) usage\n WHERE workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "executions", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - null - ] - }, - "hash": "89940a53f29b173b6a8717f057a81a4f2edad152581950fdd80d758a0d242c17" -} diff --git a/backend/.sqlx/query-899b48109ce20a8fbcf9c8e9339713dcdf4173564d388f1927dea06653c718d5.json b/backend/.sqlx/query-899b48109ce20a8fbcf9c8e9339713dcdf4173564d388f1927dea06653c718d5.json deleted file mode 100644 index f552871fff..0000000000 --- a/backend/.sqlx/query-899b48109ce20a8fbcf9c8e9339713dcdf4173564d388f1927dea06653c718d5.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE flow SET ws_error_handler_muted = $3 WHERE path = $1 AND workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "899b48109ce20a8fbcf9c8e9339713dcdf4173564d388f1927dea06653c718d5" -} diff --git a/backend/.sqlx/query-618ec69c9c78f1c9e3539d2770392e3f783f29cd2cc58c0fc87d14ecef32b467.json b/backend/.sqlx/query-8d4ad4ee75fb149c36a9f6a0c4cf5fd981473f45d1b71b4b8236e021f7c8682d.json similarity index 60% rename from backend/.sqlx/query-618ec69c9c78f1c9e3539d2770392e3f783f29cd2cc58c0fc87d14ecef32b467.json rename to backend/.sqlx/query-8d4ad4ee75fb149c36a9f6a0c4cf5fd981473f45d1b71b4b8236e021f7c8682d.json index 2aa95c8d2c..662bc5f6d6 100644 --- a/backend/.sqlx/query-618ec69c9c78f1c9e3539d2770392e3f783f29cd2cc58c0fc87d14ecef32b467.json +++ b/backend/.sqlx/query-8d4ad4ee75fb149c36a9f6a0c4cf5fd981473f45d1b71b4b8236e021f7c8682d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM _sqlx_migrations WHERE\n version=20250131115248 OR version=20250902085503 OR version=20250201145630 OR\n version=20250201145631 OR version=20250201145632", + "query": "DELETE FROM _sqlx_migrations WHERE\n version=20250131115248 OR version=20250902085503 OR version=20250201145630 OR\n version=20250201145631 OR version=20250201145632 OR version=20251006143821", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "618ec69c9c78f1c9e3539d2770392e3f783f29cd2cc58c0fc87d14ecef32b467" + "hash": "8d4ad4ee75fb149c36a9f6a0c4cf5fd981473f45d1b71b4b8236e021f7c8682d" } diff --git a/backend/.sqlx/query-4fb3881cdbb4b9e93e28f460a9b3715bdc6a52b76c89f3a3913023b13c4e085c.json b/backend/.sqlx/query-8fda0400ec5ba04a2a1469672bbb0af413027dbd39aa2c4182e3d219640d397e.json similarity index 69% rename from backend/.sqlx/query-4fb3881cdbb4b9e93e28f460a9b3715bdc6a52b76c89f3a3913023b13c4e085c.json rename to backend/.sqlx/query-8fda0400ec5ba04a2a1469672bbb0af413027dbd39aa2c4182e3d219640d397e.json index 9709a354cf..226bcb9083 100644 --- a/backend/.sqlx/query-4fb3881cdbb4b9e93e28f460a9b3715bdc6a52b76c89f3a3913023b13c4e085c.json +++ b/backend/.sqlx/query-8fda0400ec5ba04a2a1469672bbb0af413027dbd39aa2c4182e3d219640d397e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", + "query": "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = EXCLUDED.lockfile", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "4fb3881cdbb4b9e93e28f460a9b3715bdc6a52b76c89f3a3913023b13c4e085c" + "hash": "8fda0400ec5ba04a2a1469672bbb0af413027dbd39aa2c4182e3d219640d397e" } diff --git a/backend/.sqlx/query-90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71.json b/backend/.sqlx/query-90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71.json deleted file mode 100644 index 61598c0b2d..0000000000 --- a/backend/.sqlx/query-90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT tag as \"tag!\", COUNT(*) as \"count!\"\n FROM v2_as_completed_job\n WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2)\n GROUP BY tag\n ORDER BY \"count!\" DESC\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tag!", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "count!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Float8", - "Text" - ] - }, - "nullable": [ - true, - null - ] - }, - "hash": "90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71" -} diff --git a/backend/.sqlx/query-9116102c6ccad5b0d752d5d690c233dfe48062aef23072b4f4ae4ab5ca269082.json b/backend/.sqlx/query-9116102c6ccad5b0d752d5d690c233dfe48062aef23072b4f4ae4ab5ca269082.json deleted file mode 100644 index 6e58a79dd3..0000000000 --- a/backend/.sqlx/query-9116102c6ccad5b0d752d5d690c233dfe48062aef23072b4f4ae4ab5ca269082.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n postgres_trigger\n SET \n last_server_ping = now(),\n error = $1\n WHERE\n workspace_id = $2\n AND path = $3\n AND server_id = $4 \n AND enabled IS TRUE\n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "9116102c6ccad5b0d752d5d690c233dfe48062aef23072b4f4ae4ab5ca269082" -} diff --git a/backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json b/backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json deleted file mode 100644 index 14d37809d5..0000000000 --- a/backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n gcp_trigger \n SET \n server_id = $1, \n last_server_ping = now(),\n error = 'Connecting...'\n WHERE \n enabled IS TRUE \n AND workspace_id = $2 \n AND path = $3 \n AND (last_server_ping IS NULL \n OR last_server_ping < now() - INTERVAL '15 seconds'\n ) \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633" -} diff --git a/backend/.sqlx/query-92c7c961198e506426bf3f97a8ddbb34af450041c675b30b708fed3ef9e01d2d.json b/backend/.sqlx/query-92c7c961198e506426bf3f97a8ddbb34af450041c675b30b708fed3ef9e01d2d.json index bcc10fcf4a..32358798b9 100644 --- a/backend/.sqlx/query-92c7c961198e506426bf3f97a8ddbb34af450041c675b30b708fed3ef9e01d2d.json +++ b/backend/.sqlx/query-92c7c961198e506426bf3f97a8ddbb34af450041c675b30b708fed3ef9e01d2d.json @@ -25,7 +25,7 @@ "noop", "appdependencies", "deploymentcallback", - "singlescriptflow", + "singlestepflow", "flowscript", "flownode", "appscript", diff --git a/backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json b/backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json deleted file mode 100644 index acc6769652..0000000000 --- a/backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE websocket_trigger SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE enabled IS TRUE AND workspace_id = $2 AND path = $3 AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3" -} diff --git a/backend/.sqlx/query-931703a98d2ee5fb58d3380896baaee032e731db1e6bd49d991a54f49ab8fa46.json b/backend/.sqlx/query-931703a98d2ee5fb58d3380896baaee032e731db1e6bd49d991a54f49ab8fa46.json deleted file mode 100644 index 5f86a45011..0000000000 --- a/backend/.sqlx/query-931703a98d2ee5fb58d3380896baaee032e731db1e6bd49d991a54f49ab8fa46.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "create index concurrently if not exists ix_completed_job_workspace_id_started_at_new_2 ON v2_job_completed (workspace_id, started_at DESC)", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "931703a98d2ee5fb58d3380896baaee032e731db1e6bd49d991a54f49ab8fa46" -} diff --git a/backend/.sqlx/query-714fb0f66ceb536aee8cb9ae0144757b999d25870fda37fe904e09dd5c742015.json b/backend/.sqlx/query-9360d00990822f153ff09c7905ae3180f07d02f38ac12d07a5664d93f160e7ee.json similarity index 68% rename from backend/.sqlx/query-714fb0f66ceb536aee8cb9ae0144757b999d25870fda37fe904e09dd5c742015.json rename to backend/.sqlx/query-9360d00990822f153ff09c7905ae3180f07d02f38ac12d07a5664d93f160e7ee.json index 9f3f4b0c8c..c702405099 100644 --- a/backend/.sqlx/query-714fb0f66ceb536aee8cb9ae0144757b999d25870fda37fe904e09dd5c742015.json +++ b/backend/.sqlx/query-9360d00990822f153ff09c7905ae3180f07d02f38ac12d07a5664d93f160e7ee.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n route_path,\n http_method AS \"http_method: _\",\n is_async,\n workspaced_route,\n summary,\n description,\n authentication_method AS \"authentication_method: _\",\n authentication_resource_path\n FROM\n http_trigger\n WHERE\n path ~ ANY($1) AND\n route_path ~ ANY($2) AND\n workspace_id = $3\n ", + "query": "\n SELECT\n route_path,\n http_method AS \"http_method: _\",\n request_type AS \"request_type: _\",\n workspaced_route,\n summary,\n description,\n authentication_method AS \"authentication_method: _\",\n authentication_resource_path\n FROM\n http_trigger\n WHERE\n path ~ ANY($1) AND\n route_path ~ ANY($2) AND\n workspace_id = $3\n ", "describe": { "columns": [ { @@ -28,8 +28,19 @@ }, { "ordinal": 2, - "name": "is_async", - "type_info": "Bool" + "name": "request_type: _", + "type_info": { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + } }, { "ordinal": 3, @@ -89,5 +100,5 @@ true ] }, - "hash": "714fb0f66ceb536aee8cb9ae0144757b999d25870fda37fe904e09dd5c742015" + "hash": "9360d00990822f153ff09c7905ae3180f07d02f38ac12d07a5664d93f160e7ee" } diff --git a/backend/.sqlx/query-93d64930c74ccb1abdc9bda8287540a354f8eec49913aaf07fc1e737e7c93330.json b/backend/.sqlx/query-93d64930c74ccb1abdc9bda8287540a354f8eec49913aaf07fc1e737e7c93330.json deleted file mode 100644 index 5b30dd2cbd..0000000000 --- a/backend/.sqlx/query-93d64930c74ccb1abdc9bda8287540a354f8eec49913aaf07fc1e737e7c93330.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = 'nats' AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "93d64930c74ccb1abdc9bda8287540a354f8eec49913aaf07fc1e737e7c93330" -} diff --git a/backend/.sqlx/query-93f00e2d164c090a959da12cbb7e4fc6e5804fea077116d339a041c0f2b0c1fa.json b/backend/.sqlx/query-93f00e2d164c090a959da12cbb7e4fc6e5804fea077116d339a041c0f2b0c1fa.json new file mode 100644 index 0000000000..7600acb32a --- /dev/null +++ b/backend/.sqlx/query-93f00e2d164c090a959da12cbb7e4fc6e5804fea077116d339a041c0f2b0c1fa.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET flow_status = \n CASE \n WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN\n JSONB_SET(\n JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n ELSE\n JSONB_SET(JSONB_SET(\n flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n END\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [ + null + ] + }, + "hash": "93f00e2d164c090a959da12cbb7e4fc6e5804fea077116d339a041c0f2b0c1fa" +} diff --git a/backend/.sqlx/query-95d5fbae671ce95eae978193dbf3c24e93208a61e65811dd14a3820102c1ae57.json b/backend/.sqlx/query-95d5fbae671ce95eae978193dbf3c24e93208a61e65811dd14a3820102c1ae57.json new file mode 100644 index 0000000000..85a5332bac --- /dev/null +++ b/backend/.sqlx/query-95d5fbae671ce95eae978193dbf3c24e93208a61e65811dd14a3820102c1ae57.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_env WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "95d5fbae671ce95eae978193dbf3c24e93208a61e65811dd14a3820102c1ae57" +} diff --git a/backend/.sqlx/query-96c0e34708bbba29db162e7289a942addd4581dddc88663b6c2cbae87ec205fc.json b/backend/.sqlx/query-96c0e34708bbba29db162e7289a942addd4581dddc88663b6c2cbae87ec205fc.json deleted file mode 100644 index 0f6659264a..0000000000 --- a/backend/.sqlx/query-96c0e34708bbba29db162e7289a942addd4581dddc88663b6c2cbae87ec205fc.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid", - "Text", - "Jsonb" - ] - }, - "nullable": [ - null - ] - }, - "hash": "96c0e34708bbba29db162e7289a942addd4581dddc88663b6c2cbae87ec205fc" -} diff --git a/backend/.sqlx/query-97547c49f4ed07e7d07ddf2ef971abb820df9d6ec54a340cf8b1332e1052f666.json b/backend/.sqlx/query-97547c49f4ed07e7d07ddf2ef971abb820df9d6ec54a340cf8b1332e1052f666.json deleted file mode 100644 index 0a9fd45382..0000000000 --- a/backend/.sqlx/query-97547c49f4ed07e7d07ddf2ef971abb820df9d6ec54a340cf8b1332e1052f666.json +++ /dev/null @@ -1,271 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT hash, path, summary, description, content,\n created_at, archived, schema, deleted, is_template,\n extra_perms, lock, lock_error_logs, language as \"language: ScriptLang\", \n kind as \"kind: ScriptKind\", tag, draft_only, envs, concurrent_limit, \n concurrency_time_window_s, cache_ttl, dedicated_worker, \n ws_error_handler_muted, priority, timeout, delete_after_use, \n restart_unless_cancelled, concurrency_key, visible_to_runner_only,\n no_main_func, codebase, has_preprocessor, on_behalf_of_email,\n parent_hashes, assets\n FROM script WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "hash", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "summary", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "description", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "content", - "type_info": "Text" - }, - { - "ordinal": 5, - "name": "created_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 6, - "name": "archived", - "type_info": "Bool" - }, - { - "ordinal": 7, - "name": "schema", - "type_info": "Json" - }, - { - "ordinal": 8, - "name": "deleted", - "type_info": "Bool" - }, - { - "ordinal": 9, - "name": "is_template", - "type_info": "Bool" - }, - { - "ordinal": 10, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 11, - "name": "lock", - "type_info": "Text" - }, - { - "ordinal": 12, - "name": "lock_error_logs", - "type_info": "Text" - }, - { - "ordinal": 13, - "name": "language: ScriptLang", - "type_info": { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java", - "duckdb", - "ruby" - ] - } - } - } - }, - { - "ordinal": 14, - "name": "kind: ScriptKind", - "type_info": { - "Custom": { - "name": "script_kind", - "kind": { - "Enum": [ - "script", - "trigger", - "failure", - "command", - "approval", - "preprocessor" - ] - } - } - } - }, - { - "ordinal": 15, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 16, - "name": "draft_only", - "type_info": "Bool" - }, - { - "ordinal": 17, - "name": "envs", - "type_info": "VarcharArray" - }, - { - "ordinal": 18, - "name": "concurrent_limit", - "type_info": "Int4" - }, - { - "ordinal": 19, - "name": "concurrency_time_window_s", - "type_info": "Int4" - }, - { - "ordinal": 20, - "name": "cache_ttl", - "type_info": "Int4" - }, - { - "ordinal": 21, - "name": "dedicated_worker", - "type_info": "Bool" - }, - { - "ordinal": 22, - "name": "ws_error_handler_muted", - "type_info": "Bool" - }, - { - "ordinal": 23, - "name": "priority", - "type_info": "Int2" - }, - { - "ordinal": 24, - "name": "timeout", - "type_info": "Int4" - }, - { - "ordinal": 25, - "name": "delete_after_use", - "type_info": "Bool" - }, - { - "ordinal": 26, - "name": "restart_unless_cancelled", - "type_info": "Bool" - }, - { - "ordinal": 27, - "name": "concurrency_key", - "type_info": "Varchar" - }, - { - "ordinal": 28, - "name": "visible_to_runner_only", - "type_info": "Bool" - }, - { - "ordinal": 29, - "name": "no_main_func", - "type_info": "Bool" - }, - { - "ordinal": 30, - "name": "codebase", - "type_info": "Varchar" - }, - { - "ordinal": 31, - "name": "has_preprocessor", - "type_info": "Bool" - }, - { - "ordinal": 32, - "name": "on_behalf_of_email", - "type_info": "Text" - }, - { - "ordinal": 33, - "name": "parent_hashes", - "type_info": "Int8Array" - }, - { - "ordinal": 34, - "name": "assets", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - true, - false, - true, - false, - true, - true, - false, - false, - true, - true, - true, - true, - true, - true, - true, - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true - ] - }, - "hash": "97547c49f4ed07e7d07ddf2ef971abb820df9d6ec54a340cf8b1332e1052f666" -} diff --git a/backend/.sqlx/query-97e3a1439202e13e739ad2e3f22b3a21d0c9b0e57d7d35326753e8f6a804d4f8.json b/backend/.sqlx/query-97e3a1439202e13e739ad2e3f22b3a21d0c9b0e57d7d35326753e8f6a804d4f8.json new file mode 100644 index 0000000000..5ab1af21c5 --- /dev/null +++ b/backend/.sqlx/query-97e3a1439202e13e739ad2e3f22b3a21d0c9b0e57d7d35326753e8f6a804d4f8.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Name" + ] + }, + "nullable": [ + null + ] + }, + "hash": "97e3a1439202e13e739ad2e3f22b3a21d0c9b0e57d7d35326753e8f6a804d4f8" +} diff --git a/backend/.sqlx/query-999edc6f54a9efb6dc6237992dad59f418ed6fc2a98ddb9bbc33dce5f029d904.json b/backend/.sqlx/query-999edc6f54a9efb6dc6237992dad59f418ed6fc2a98ddb9bbc33dce5f029d904.json deleted file mode 100644 index 8748572af7..0000000000 --- a/backend/.sqlx/query-999edc6f54a9efb6dc6237992dad59f418ed6fc2a98ddb9bbc33dce5f029d904.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n gcp_trigger \n SET\n last_server_ping = NULL \n WHERE \n workspace_id = $1 \n AND path = $2 \n AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "999edc6f54a9efb6dc6237992dad59f418ed6fc2a98ddb9bbc33dce5f029d904" -} diff --git a/backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json b/backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json deleted file mode 100644 index f38b293acb..0000000000 --- a/backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE last_client_ping > NOW() - INTERVAL '10 seconds' AND workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket' AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46" -} diff --git a/backend/.sqlx/query-99c289a8bcf87588ecb89575f66e1fbfce74dd6b69a8a039714a02ce7558a1b3.json b/backend/.sqlx/query-99c289a8bcf87588ecb89575f66e1fbfce74dd6b69a8a039714a02ce7558a1b3.json new file mode 100644 index 0000000000..6e9dd142f6 --- /dev/null +++ b/backend/.sqlx/query-99c289a8bcf87588ecb89575f66e1fbfce74dd6b69a8a039714a02ce7558a1b3.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT instance_group.id, COALESCE(instance_group.scim_display_name, instance_group.name) as display_name\n FROM email_to_igroup\n JOIN instance_group ON instance_group.name = email_to_igroup.igroup\n WHERE email_to_igroup.email = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "display_name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + null + ] + }, + "hash": "99c289a8bcf87588ecb89575f66e1fbfce74dd6b69a8a039714a02ce7558a1b3" +} diff --git a/backend/.sqlx/query-99e11c04bcc436ec7a75f46365423c85a6e6490c2d9e1dc6ac39112d763f0f75.json b/backend/.sqlx/query-99e11c04bcc436ec7a75f46365423c85a6e6490c2d9e1dc6ac39112d763f0f75.json new file mode 100644 index 0000000000..6fb4205e32 --- /dev/null +++ b/backend/.sqlx/query-99e11c04bcc436ec7a75f46365423c85a6e6490c2d9e1dc6ac39112d763f0f75.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM app_version WHERE app_id = (SELECT id FROM app WHERE path = $1 AND workspace_id = $2) ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "99e11c04bcc436ec7a75f46365423c85a6e6490c2d9e1dc6ac39112d763f0f75" +} diff --git a/backend/.sqlx/query-e40f7e0b61567f948bfea0b6f50518564634885ccc2c0d30ccca79fc13bdcf07.json b/backend/.sqlx/query-99e6bffe177e69448b09e82b30d24af00edc86a9bc498f319c1e5bee55d77a8a.json similarity index 60% rename from backend/.sqlx/query-e40f7e0b61567f948bfea0b6f50518564634885ccc2c0d30ccca79fc13bdcf07.json rename to backend/.sqlx/query-99e6bffe177e69448b09e82b30d24af00edc86a9bc498f319c1e5bee55d77a8a.json index be7ab6923c..c5a256e0d7 100644 --- a/backend/.sqlx/query-e40f7e0b61567f948bfea0b6f50518564634885ccc2c0d30ccca79fc13bdcf07.json +++ b/backend/.sqlx/query-99e6bffe177e69448b09e82b30d24af00edc86a9bc498f319c1e5bee55d77a8a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO tutorial_progress VALUES ($2, $1::bigint::bit(64)) ON CONFLICT (email) DO UPDATE SET progress = $1::bigint::bit(64)", + "query": "INSERT INTO tutorial_progress VALUES ($2, $1::bigint::bit(64)) ON CONFLICT (email) DO UPDATE SET progress = EXCLUDED.progress", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "e40f7e0b61567f948bfea0b6f50518564634885ccc2c0d30ccca79fc13bdcf07" + "hash": "99e6bffe177e69448b09e82b30d24af00edc86a9bc498f319c1e5bee55d77a8a" } diff --git a/backend/.sqlx/query-9c0bbd44902d8eee393236f7c2372b273d14a093bb29ec12dda8bbfaecd49a35.json b/backend/.sqlx/query-9c0bbd44902d8eee393236f7c2372b273d14a093bb29ec12dda8bbfaecd49a35.json new file mode 100644 index 0000000000..5740a1bda5 --- /dev/null +++ b/backend/.sqlx/query-9c0bbd44902d8eee393236f7c2372b273d14a093bb29ec12dda8bbfaecd49a35.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH result_stream AS (\n SELECT \n string_agg(stream, '' order by idx asc) as stream, \n job_id, \n max(idx) + 1 as offset \n FROM job_result_stream_v2\n WHERE job_id = $2 AND idx >= $3\n GROUP BY job_id\n )\n SELECT \n jc.result as \"result: sqlx::types::Json>\",\n v2_job.tag,\n v2_job_queue.running as \"running: Option\",\n rs.stream AS \"result_stream: Option\",\n rs.offset AS stream_offset,\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM v2_job\n LEFT JOIN v2_job_queue USING (id)\n LEFT JOIN v2_job_completed jc USING (id)\n LEFT JOIN v2_job_status js USING (id)\n LEFT JOIN result_stream rs ON rs.job_id = $2\n WHERE v2_job.id = $2 AND v2_job.workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "running: Option", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "result_stream: Option", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "stream_offset", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "stream_job", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Int4", + "Bool" + ] + }, + "nullable": [ + true, + false, + false, + null, + null, + null + ] + }, + "hash": "9c0bbd44902d8eee393236f7c2372b273d14a093bb29ec12dda8bbfaecd49a35" +} diff --git a/backend/.sqlx/query-2d5f58dd2aff3bd49f3891ae76df23e2aa39891931516426f65b229314a0cee1.json b/backend/.sqlx/query-9c3ddb90295db7d6afcbdb077f017950620e753dde97c8d2d88cd60ff8c3f339.json similarity index 72% rename from backend/.sqlx/query-2d5f58dd2aff3bd49f3891ae76df23e2aa39891931516426f65b229314a0cee1.json rename to backend/.sqlx/query-9c3ddb90295db7d6afcbdb077f017950620e753dde97c8d2d88cd60ff8c3f339.json index 9d2440990e..96bc0d74f4 100644 --- a/backend/.sqlx/query-2d5f58dd2aff3bd49f3891ae76df23e2aa39891931516426f65b229314a0cee1.json +++ b/backend/.sqlx/query-9c3ddb90295db7d6afcbdb077f017950620e753dde97c8d2d88cd60ff8c3f339.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id as workspace, path, summary, description, schema FROM script as o \n WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1 AND archived = false) \n AND workspace_id = $1 and archived = false", + "query": "SELECT workspace_id as workspace, path, summary, description, schema FROM script as o\n WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1 AND archived = false)\n AND workspace_id = $1 and archived = false", "describe": { "columns": [ { @@ -42,5 +42,5 @@ true ] }, - "hash": "2d5f58dd2aff3bd49f3891ae76df23e2aa39891931516426f65b229314a0cee1" + "hash": "9c3ddb90295db7d6afcbdb077f017950620e753dde97c8d2d88cd60ff8c3f339" } diff --git a/backend/.sqlx/query-9c72b2962d0919353cfe5af710e857d432dff44e343b8f0610208d42ff5afd14.json b/backend/.sqlx/query-9c72b2962d0919353cfe5af710e857d432dff44e343b8f0610208d42ff5afd14.json deleted file mode 100644 index a2f8a647c5..0000000000 --- a/backend/.sqlx/query-9c72b2962d0919353cfe5af710e857d432dff44e343b8f0610208d42ff5afd14.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO global_settings (name, value)\n VALUES ('teams', $1)\n ON CONFLICT (name) DO UPDATE SET value = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "9c72b2962d0919353cfe5af710e857d432dff44e343b8f0610208d42ff5afd14" -} diff --git a/backend/.sqlx/query-23eb4d45bf2df21e22fc6c9590b96b0a7dbdd27f85c7d886eded79b3af83731a.json b/backend/.sqlx/query-9ebf262393fc4a29e8f09b304dd99e786fe78dd21721e8f54dc943dd571a7e08.json similarity index 65% rename from backend/.sqlx/query-23eb4d45bf2df21e22fc6c9590b96b0a7dbdd27f85c7d886eded79b3af83731a.json rename to backend/.sqlx/query-9ebf262393fc4a29e8f09b304dd99e786fe78dd21721e8f54dc943dd571a7e08.json index ecaefdeb62..53c7562d03 100644 --- a/backend/.sqlx/query-23eb4d45bf2df21e22fc6c9590b96b0a7dbdd27f85c7d886eded79b3af83731a.json +++ b/backend/.sqlx/query-9ebf262393fc4a29e8f09b304dd99e786fe78dd21721e8f54dc943dd571a7e08.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO websocket_trigger (\n workspace_id,\n path,\n url,\n script_path,\n is_flow,\n enabled,\n filters,\n initial_messages,\n url_runnable_args,\n edited_by,\n can_return_message,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13, $14, $15\n )\n ", + "query": "\n INSERT INTO websocket_trigger (\n workspace_id,\n path,\n url,\n script_path,\n is_flow,\n enabled,\n filters,\n initial_messages,\n url_runnable_args,\n edited_by,\n can_return_message,\n can_return_error_result,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), $14, $15, $16\n )\n ", "describe": { "columns": [], "parameters": { @@ -16,6 +16,7 @@ "Jsonb", "Varchar", "Bool", + "Bool", "Varchar", "Varchar", "Jsonb", @@ -24,5 +25,5 @@ }, "nullable": [] }, - "hash": "23eb4d45bf2df21e22fc6c9590b96b0a7dbdd27f85c7d886eded79b3af83731a" + "hash": "9ebf262393fc4a29e8f09b304dd99e786fe78dd21721e8f54dc943dd571a7e08" } diff --git a/backend/.sqlx/query-a0af7dc507778ab23d5b1615e2a03278de05f4cd90fb1c31d7281f8681c7f85d.json b/backend/.sqlx/query-a0af7dc507778ab23d5b1615e2a03278de05f4cd90fb1c31d7281f8681c7f85d.json new file mode 100644 index 0000000000..2741840d4b --- /dev/null +++ b/backend/.sqlx/query-a0af7dc507778ab23d5b1615e2a03278de05f4cd90fb1c31d7281f8681c7f85d.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE job_logs SET logs = $1, log_offset = $2,\n log_file_index = array_append(coalesce(log_file_index, array[]::text[]), $3)\n WHERE workspace_id = $4 AND job_id = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int4", + "Text", + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "a0af7dc507778ab23d5b1615e2a03278de05f4cd90fb1c31d7281f8681c7f85d" +} diff --git a/backend/.sqlx/query-a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887.json b/backend/.sqlx/query-a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887.json deleted file mode 100644 index 1afc61978e..0000000000 --- a/backend/.sqlx/query-a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_status (id, flow_status) SELECT unnest($1::uuid[]), $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887" -} diff --git a/backend/.sqlx/query-6d134b137ae81534e145fc5b6474cf963ee26a3ad3a0a3d8dc064cb14c8fd9a6.json b/backend/.sqlx/query-a38df5d7dc4577c715d9acdaf87c38535ad388b1948a95efafd71135cfe5e3a6.json similarity index 83% rename from backend/.sqlx/query-6d134b137ae81534e145fc5b6474cf963ee26a3ad3a0a3d8dc064cb14c8fd9a6.json rename to backend/.sqlx/query-a38df5d7dc4577c715d9acdaf87c38535ad388b1948a95efafd71135cfe5e3a6.json index 03351cdeab..7e5e0043ab 100644 --- a/backend/.sqlx/query-6d134b137ae81534e145fc5b6474cf963ee26a3ad3a0a3d8dc064cb14c8fd9a6.json +++ b/backend/.sqlx/query-a38df5d7dc4577c715d9acdaf87c38535ad388b1948a95efafd71135cfe5e3a6.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, worker_group, event_type::text, desired_workers, reason, applied_at FROM autoscaling_event WHERE worker_group = $1 ORDER BY applied_at DESC LIMIT 5", + "query": "SELECT id, worker_group, event_type::text, desired_workers, reason, applied_at FROM autoscaling_event WHERE worker_group = $1 ORDER BY applied_at DESC LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -36,7 +36,9 @@ ], "parameters": { "Left": [ - "Text" + "Text", + "Int8", + "Int8" ] }, "nullable": [ @@ -48,5 +50,5 @@ false ] }, - "hash": "6d134b137ae81534e145fc5b6474cf963ee26a3ad3a0a3d8dc064cb14c8fd9a6" + "hash": "a38df5d7dc4577c715d9acdaf87c38535ad388b1948a95efafd71135cfe5e3a6" } diff --git a/backend/.sqlx/query-11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a.json b/backend/.sqlx/query-a3d18ae5e5125940ae0d6af315e2dbb17c739415850e7c67f8575fb983c295fe.json similarity index 55% rename from backend/.sqlx/query-11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a.json rename to backend/.sqlx/query-a3d18ae5e5125940ae0d6af315e2dbb17c739415850e7c67f8575fb983c295fe.json index 1afb036030..d66a3aa9f0 100644 --- a/backend/.sqlx/query-11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a.json +++ b/backend/.sqlx/query-a3d18ae5e5125940ae0d6af315e2dbb17c739415850e7c67f8575fb983c295fe.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COUNT(id) FROM v2_as_queue WHERE running = true AND email = $1", + "query": "SELECT COUNT(j.id) FROM v2_job_queue q JOIN v2_job j USING (id) WHERE q.running = true AND j.permissioned_as_email = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a" + "hash": "a3d18ae5e5125940ae0d6af315e2dbb17c739415850e7c67f8575fb983c295fe" } diff --git a/backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json b/backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json new file mode 100644 index 0000000000..4e3b39403f --- /dev/null +++ b/backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id, app_path)\n SELECT flow_path, runnable_path, script_hash, runnable_is_flow, $1, app_path\n FROM workspace_runnable_dependencies\n WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e" +} diff --git a/backend/.sqlx/query-20fcdcd2674a52ee9bd8d1de518d6bce075f20bcd5d2328f183d8a59331f6bb1.json b/backend/.sqlx/query-a5bf005e0f7c9a86a136e049445de059481091ac173414418b8678f6beadf2ac.json similarity index 75% rename from backend/.sqlx/query-20fcdcd2674a52ee9bd8d1de518d6bce075f20bcd5d2328f183d8a59331f6bb1.json rename to backend/.sqlx/query-a5bf005e0f7c9a86a136e049445de059481091ac173414418b8678f6beadf2ac.json index 824335990f..6317641989 100644 --- a/backend/.sqlx/query-20fcdcd2674a52ee9bd8d1de518d6bce075f20bcd5d2328f183d8a59331f6bb1.json +++ b/backend/.sqlx/query-a5bf005e0f7c9a86a136e049445de059481091ac173414418b8678f6beadf2ac.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO resource\n (workspace_id, path, value, resource_type, created_by, edited_at)\n VALUES ($1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, path) DO UPDATE SET value = $3, edited_at = now()", + "query": "INSERT INTO resource\n (workspace_id, path, value, resource_type, created_by, edited_at)\n VALUES ($1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, path) DO UPDATE SET value = EXCLUDED.value, edited_at = now()", "describe": { "columns": [], "parameters": { @@ -14,5 +14,5 @@ }, "nullable": [] }, - "hash": "20fcdcd2674a52ee9bd8d1de518d6bce075f20bcd5d2328f183d8a59331f6bb1" + "hash": "a5bf005e0f7c9a86a136e049445de059481091ac173414418b8678f6beadf2ac" } diff --git a/backend/.sqlx/query-a5fbef9db2308920ea26f6154f0f3490f88ced636c9ebdecba2205f964b1d0f1.json b/backend/.sqlx/query-a5fbef9db2308920ea26f6154f0f3490f88ced636c9ebdecba2205f964b1d0f1.json deleted file mode 100644 index 219558ac68..0000000000 --- a/backend/.sqlx/query-a5fbef9db2308920ea26f6154f0f3490f88ced636c9ebdecba2205f964b1d0f1.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n sqs_trigger \n SET\n last_server_ping = NULL \n WHERE \n workspace_id = $1 \n AND path = $2 \n AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "a5fbef9db2308920ea26f6154f0f3490f88ced636c9ebdecba2205f964b1d0f1" -} diff --git a/backend/.sqlx/query-a6608d47b96d851eb7b04d2e4b472889ff257db0f0e3e9252adda2e4ef2039d6.json b/backend/.sqlx/query-a6608d47b96d851eb7b04d2e4b472889ff257db0f0e3e9252adda2e4ef2039d6.json deleted file mode 100644 index 8fea4c40ec..0000000000 --- a/backend/.sqlx/query-a6608d47b96d851eb7b04d2e4b472889ff257db0f0e3e9252adda2e4ef2039d6.json +++ /dev/null @@ -1,249 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n cj.parent_job,\n cj.created_by,\n cj.duration_ms,\n cj.success,\n cj.script_hash,\n cj.script_path,\n cj.args,\n cj.result,\n cj.deleted,\n cj.canceled,\n cj.canceled_by,\n cj.canceled_reason,\n cj.job_kind,\n cj.schedule_path,\n cj.permissioned_as,\n cj.is_flow_step,\n cj.language,\n cj.is_skipped,\n cj.email,\n cj.visible_to_owner,\n cj.mem_peak,\n cj.tag,\n cj.created_at,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(cj.created_by), 0) + \n COALESCE(LENGTH(cj.script_path), 0) + \n COALESCE(LENGTH(cj.args::text), 0) + \n COALESCE(LENGTH(cj.result::text), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_as_completed_job AS cj\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE cj.created_at < $1\n ORDER BY created_at ASC\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT \n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY created_at ASC \n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes,\n ROW_NUMBER() OVER (ORDER BY created_at ASC) AS row_num\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option\",\n script_path,\n args AS \"args: sqlx::types::Json>>\",\n result AS \"result: sqlx::types::Json>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n created_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $2 OR row_num = 1\n ORDER BY created_at ASC", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id!", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "workspace_id!", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "parent_job", - "type_info": "Uuid" - }, - { - "ordinal": 3, - "name": "created_by!", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "duration_ms!", - "type_info": "Int8" - }, - { - "ordinal": 5, - "name": "success!", - "type_info": "Bool" - }, - { - "ordinal": 6, - "name": "script_hash!: Option", - "type_info": "Int8" - }, - { - "ordinal": 7, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "args: sqlx::types::Json>>", - "type_info": "Jsonb" - }, - { - "ordinal": 9, - "name": "result: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 10, - "name": "deleted!", - "type_info": "Bool" - }, - { - "ordinal": 11, - "name": "canceled!", - "type_info": "Bool" - }, - { - "ordinal": 12, - "name": "canceled_by", - "type_info": "Varchar" - }, - { - "ordinal": 13, - "name": "canceled_reason", - "type_info": "Text" - }, - { - "ordinal": 14, - "name": "job_kind!: JobKind", - "type_info": { - "Custom": { - "name": "job_kind", - "kind": { - "Enum": [ - "script", - "preview", - "flow", - "dependencies", - "flowpreview", - "script_hub", - "identity", - "flowdependencies", - "http", - "graphql", - "postgresql", - "noop", - "appdependencies", - "deploymentcallback", - "singlescriptflow", - "flowscript", - "flownode", - "appscript", - "aiagent" - ] - } - } - } - }, - { - "ordinal": 15, - "name": "schedule_path", - "type_info": "Varchar" - }, - { - "ordinal": 16, - "name": "permissioned_as!", - "type_info": "Varchar" - }, - { - "ordinal": 17, - "name": "is_flow_step!", - "type_info": "Bool" - }, - { - "ordinal": 18, - "name": "language: ScriptLang", - "type_info": { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java", - "duckdb", - "ruby" - ] - } - } - } - }, - { - "ordinal": 19, - "name": "is_skipped!", - "type_info": "Bool" - }, - { - "ordinal": 20, - "name": "email!", - "type_info": "Varchar" - }, - { - "ordinal": 21, - "name": "visible_to_owner!", - "type_info": "Bool" - }, - { - "ordinal": 22, - "name": "mem_peak", - "type_info": "Int4" - }, - { - "ordinal": 23, - "name": "tag!", - "type_info": "Varchar" - }, - { - "ordinal": 24, - "name": "created_at!", - "type_info": "Timestamptz" - }, - { - "ordinal": 25, - "name": "started_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 26, - "name": "logs", - "type_info": "Text" - }, - { - "ordinal": 27, - "name": "log_offset?", - "type_info": "Int4" - }, - { - "ordinal": 28, - "name": "log_file_index", - "type_info": "TextArray" - } - ], - "parameters": { - "Left": [ - "Timestamptz", - "Int8" - ] - }, - "nullable": [ - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - false, - true - ] - }, - "hash": "a6608d47b96d851eb7b04d2e4b472889ff257db0f0e3e9252adda2e4ef2039d6" -} diff --git a/backend/.sqlx/query-a6a973dcd92d2e40fd9a1c1be42052fcd350bd47ee4f63832448b6e6f0f472f0.json b/backend/.sqlx/query-a6a973dcd92d2e40fd9a1c1be42052fcd350bd47ee4f63832448b6e6f0f472f0.json new file mode 100644 index 0000000000..1c396d38fd --- /dev/null +++ b/backend/.sqlx/query-a6a973dcd92d2e40fd9a1c1be42052fcd350bd47ee4f63832448b6e6f0f472f0.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow_version\n (workspace_id, path, value, schema, created_by)\n\n SELECT workspace_id, path, value, schema, created_by\n FROM flow_version WHERE path = $1 AND workspace_id = $2 AND id = $3\n\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a6a973dcd92d2e40fd9a1c1be42052fcd350bd47ee4f63832448b6e6f0f472f0" +} diff --git a/backend/.sqlx/query-a6d1b80e1b407610987c98521f8e36dc8e96a63c4690721ae0bc169a3d83aff1.json b/backend/.sqlx/query-a6d1b80e1b407610987c98521f8e36dc8e96a63c4690721ae0bc169a3d83aff1.json deleted file mode 100644 index 63301d439c..0000000000 --- a/backend/.sqlx/query-a6d1b80e1b407610987c98521f8e36dc8e96a63c4690721ae0bc169a3d83aff1.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Jsonb", - "Int4", - "Bool", - "Timestamptz" - ] - }, - "nullable": [] - }, - "hash": "a6d1b80e1b407610987c98521f8e36dc8e96a63c4690721ae0bc169a3d83aff1" -} diff --git a/backend/.sqlx/query-a72e7fb55d7268fe1ea40015a4a84b12dd961ba732772d8f6c59d18fe05f285d.json b/backend/.sqlx/query-a72e7fb55d7268fe1ea40015a4a84b12dd961ba732772d8f6c59d18fe05f285d.json new file mode 100644 index 0000000000..af4ada67f2 --- /dev/null +++ b/backend/.sqlx/query-a72e7fb55d7268fe1ea40015a4a84b12dd961ba732772d8f6c59d18fe05f285d.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (\n SELECT 1 FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $2\n AND (j.kind = 'appscript' OR j.kind = 'preview')\n AND j.created_by = 'anonymous'\n AND c.started_at > now() - interval '3 hours'\n AND j.runnable_path LIKE $3 || '/%'\n AND c.result @> ('{\"s3\":\"' || $1 || '\"}')::jsonb\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a72e7fb55d7268fe1ea40015a4a84b12dd961ba732772d8f6c59d18fe05f285d" +} diff --git a/backend/.sqlx/query-a739af2f72e117acc58374f6ed44f8223efa4826d6c10639f98e624474b247a3.json b/backend/.sqlx/query-a739af2f72e117acc58374f6ed44f8223efa4826d6c10639f98e624474b247a3.json new file mode 100644 index 0000000000..8a64d7c0a6 --- /dev/null +++ b/backend/.sqlx/query-a739af2f72e117acc58374f6ed44f8223efa4826d6c10639f98e624474b247a3.json @@ -0,0 +1,77 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, conversation_id, message_type as \"message_type: MessageType\", content, job_id, created_at, step_name, success\n FROM (\n SELECT id, conversation_id, message_type, content, job_id, created_at, step_name, success\n FROM flow_conversation_message\n WHERE conversation_id = $1\n ORDER BY created_at DESC, CASE WHEN message_type = 'user' THEN 0 ELSE 1 END\n LIMIT $2 OFFSET $3\n ) AS messages\n ORDER BY created_at ASC, CASE WHEN message_type = 'user' THEN 0 ELSE 1 END\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "conversation_id", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "message_type: MessageType", + "type_info": { + "Custom": { + "name": "message_type", + "kind": { + "Enum": [ + "user", + "assistant", + "tool" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "content", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 5, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "step_name", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "success", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + false, + true, + false + ] + }, + "hash": "a739af2f72e117acc58374f6ed44f8223efa4826d6c10639f98e624474b247a3" +} diff --git a/backend/.sqlx/query-a76eec5797ca8f97e63ed5542bf03873e7dfd1cf9fe984c769afb5a8bdb48d49.json b/backend/.sqlx/query-a76eec5797ca8f97e63ed5542bf03873e7dfd1cf9fe984c769afb5a8bdb48d49.json new file mode 100644 index 0000000000..ddf7d06e68 --- /dev/null +++ b/backend/.sqlx/query-a76eec5797ca8f97e63ed5542bf03873e7dfd1cf9fe984c769afb5a8bdb48d49.json @@ -0,0 +1,49 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH result_stream AS (\n SELECT \n string_agg(stream, '' order by idx asc) as stream, \n job_id, \n max(idx) + 1 as offset \n FROM job_result_stream_v2\n WHERE job_id = $1 AND idx >= $3\n GROUP BY job_id\n )\n SELECT\n COALESCE(jc.result, jc.result) as \"result: sqlx::types::Json>\",\n jq.running as \"running: Option\",\n rs.stream AS \"result_stream: Option\",\n rs.offset AS stream_offset,\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM (\n SELECT $1::uuid as job_id, $2::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN v2_job_queue jq ON jq.id = base.job_id AND jq.workspace_id = base.workspace_id\n LEFT JOIN v2_job_status js ON js.id = base.job_id\n LEFT JOIN result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "running: Option", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "result_stream: Option", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "stream_offset", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "stream_job", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int4", + "Bool" + ] + }, + "nullable": [ + null, + false, + null, + null, + null + ] + }, + "hash": "a76eec5797ca8f97e63ed5542bf03873e7dfd1cf9fe984c769afb5a8bdb48d49" +} diff --git a/backend/.sqlx/query-a7a23229d6915d7fdeea8073d31be6e7d9f7a8581bdbb5914f2c5b49f37dbc36.json b/backend/.sqlx/query-a7a23229d6915d7fdeea8073d31be6e7d9f7a8581bdbb5914f2c5b49f37dbc36.json deleted file mode 100644 index b34c87ca4c..0000000000 --- a/backend/.sqlx/query-a7a23229d6915d7fdeea8073d31be6e7d9f7a8581bdbb5914f2c5b49f37dbc36.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n http_trigger \n SET \n wrap_body = $1,\n raw_string = $2,\n authentication_resource_path = $3,\n script_path = $4, \n path = $5, \n is_flow = $6, \n http_method = $7, \n static_asset_config = $8, \n edited_by = $9, \n email = $10, \n is_async = $11, \n authentication_method = $12, \n summary = $13,\n description = $14,\n edited_at = now(), \n is_static_website = $15,\n error_handler_path = $16,\n error_handler_args = $17,\n retry = $18\n WHERE \n workspace_id = $19 AND \n path = $20\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Bool", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Bool", - { - "Custom": { - "name": "http_method", - "kind": { - "Enum": [ - "get", - "post", - "put", - "delete", - "patch" - ] - } - } - }, - "Jsonb", - "Varchar", - "Varchar", - "Bool", - { - "Custom": { - "name": "authentication_method", - "kind": { - "Enum": [ - "none", - "windmill", - "api_key", - "basic_http", - "custom_script", - "signature" - ] - } - } - }, - "Varchar", - "Text", - "Bool", - "Varchar", - "Jsonb", - "Jsonb", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "a7a23229d6915d7fdeea8073d31be6e7d9f7a8581bdbb5914f2c5b49f37dbc36" -} diff --git a/backend/.sqlx/query-a7ffc5b983d365159ef379ec6a2ab0dc7217d1b3f86f362c213982984d2cf652.json b/backend/.sqlx/query-a7ffc5b983d365159ef379ec6a2ab0dc7217d1b3f86f362c213982984d2cf652.json deleted file mode 100644 index 03b05af6fb..0000000000 --- a/backend/.sqlx/query-a7ffc5b983d365159ef379ec6a2ab0dc7217d1b3f86f362c213982984d2cf652.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n last_server_ping = now(), \n error = $1 \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'gcp' AND \n server_id = $5 AND \n last_client_ping > NOW() - INTERVAL '10 seconds' \n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "a7ffc5b983d365159ef379ec6a2ab0dc7217d1b3f86f362c213982984d2cf652" -} diff --git a/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json b/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json deleted file mode 100644 index e07a3ccdd7..0000000000 --- a/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job (id, runnable_id, runnable_path, kind, script_lang, tag, created_by, permissioned_as, permissioned_as_email, workspace_id, raw_flow) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, 1)) RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Int8", - "Varchar", - { - "Custom": { - "name": "job_kind", - "kind": { - "Enum": [ - "script", - "preview", - "flow", - "dependencies", - "flowpreview", - "script_hub", - "identity", - "flowdependencies", - "http", - "graphql", - "postgresql", - "noop", - "appdependencies", - "deploymentcallback", - "singlescriptflow", - "flowscript", - "flownode", - "appscript", - "aiagent" - ] - } - } - }, - { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java", - "duckdb", - "ruby" - ] - } - } - }, - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Jsonb" - ] - }, - "nullable": [ - false - ] - }, - "hash": "ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338" -} diff --git a/backend/.sqlx/query-abaae3dde751a41b2dbb7856ece1c840d0ea8d59346ed9e88f6f609edb543d7e.json b/backend/.sqlx/query-abaae3dde751a41b2dbb7856ece1c840d0ea8d59346ed9e88f6f609edb543d7e.json new file mode 100644 index 0000000000..c68cdc0977 --- /dev/null +++ b/backend/.sqlx/query-abaae3dde751a41b2dbb7856ece1c840d0ea8d59346ed9e88f6f609edb543d7e.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO app_bundles (app_version_id, w_id, file_type, data) VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Varchar", + "Varchar", + "Bytea" + ] + }, + "nullable": [] + }, + "hash": "abaae3dde751a41b2dbb7856ece1c840d0ea8d59346ed9e88f6f609edb543d7e" +} diff --git a/backend/.sqlx/query-ac5dd4a4d7991159e053f1c11bce03a6d51c24fc9a2d8ac872aaf41a20af5045.json b/backend/.sqlx/query-ac5dd4a4d7991159e053f1c11bce03a6d51c24fc9a2d8ac872aaf41a20af5045.json new file mode 100644 index 0000000000..8de90d69d7 --- /dev/null +++ b/backend/.sqlx/query-ac5dd4a4d7991159e053f1c11bce03a6d51c24fc9a2d8ac872aaf41a20af5045.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE v2_job_status SET flow_status = \n JSONB_SET(JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6)\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ac5dd4a4d7991159e053f1c11bce03a6d51c24fc9a2d8ac872aaf41a20af5045" +} diff --git a/backend/.sqlx/query-ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30.json b/backend/.sqlx/query-ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30.json new file mode 100644 index 0000000000..df39f8f868 --- /dev/null +++ b/backend/.sqlx/query-ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30.json @@ -0,0 +1,45 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = $5 AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email" + ] + } + } + } + ] + }, + "nullable": [ + null + ] + }, + "hash": "ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30" +} diff --git a/backend/.sqlx/query-add48c8e7c6fa2c549ad6293cbee22889d35e919d3267c1d2a265d868fa8a7d1.json b/backend/.sqlx/query-add48c8e7c6fa2c549ad6293cbee22889d35e919d3267c1d2a265d868fa8a7d1.json deleted file mode 100644 index 0468efc99f..0000000000 --- a/backend/.sqlx/query-add48c8e7c6fa2c549ad6293cbee22889d35e919d3267c1d2a265d868fa8a7d1.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE nats_trigger SET enabled = FALSE, error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "add48c8e7c6fa2c549ad6293cbee22889d35e919d3267c1d2a265d868fa8a7d1" -} diff --git a/backend/.sqlx/query-b1a9a433e577133869c067b2ce383fc6ce4e9df307feb5fd3edc0d1276d61ff1.json b/backend/.sqlx/query-b1a9a433e577133869c067b2ce383fc6ce4e9df307feb5fd3edc0d1276d61ff1.json new file mode 100644 index 0000000000..1818efc0c0 --- /dev/null +++ b/backend/.sqlx/query-b1a9a433e577133869c067b2ce383fc6ce4e9df307feb5fd3edc0d1276d61ff1.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, step_name, success)\n VALUES ($1, $2, $3, $4, $5, $6)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + { + "Custom": { + "name": "message_type", + "kind": { + "Enum": [ + "user", + "assistant", + "tool" + ] + } + } + }, + "Text", + "Uuid", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "b1a9a433e577133869c067b2ce383fc6ce4e9df307feb5fd3edc0d1276d61ff1" +} diff --git a/backend/.sqlx/query-b223c56f55a138abef8c8cb2df40472a87b7e267a143709728b09ac063c33b96.json b/backend/.sqlx/query-b223c56f55a138abef8c8cb2df40472a87b7e267a143709728b09ac063c33b96.json deleted file mode 100644 index f542ec075a..0000000000 --- a/backend/.sqlx/query-b223c56f55a138abef8c8cb2df40472a87b7e267a143709728b09ac063c33b96.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = 'kafka' AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "b223c56f55a138abef8c8cb2df40472a87b7e267a143709728b09ac063c33b96" -} diff --git a/backend/.sqlx/query-b344ba5a32ec873181390e205e16356f1b79bd994a4bd1a8655dbe17bd1e4a30.json b/backend/.sqlx/query-b344ba5a32ec873181390e205e16356f1b79bd994a4bd1a8655dbe17bd1e4a30.json new file mode 100644 index 0000000000..1874fa8f91 --- /dev/null +++ b/backend/.sqlx/query-b344ba5a32ec873181390e205e16356f1b79bd994a4bd1a8655dbe17bd1e4a30.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT ws.ducklake->'ducklakes' AS ducklake_name\n FROM workspace_settings ws\n WHERE ws.workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ducklake_name", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b344ba5a32ec873181390e205e16356f1b79bd994a4bd1a8655dbe17bd1e4a30" +} diff --git a/backend/.sqlx/query-b3c02fd225a6aa78785d466e7f033b38deb9c7fa17bd3836c9ad8884f27be84a.json b/backend/.sqlx/query-b3c02fd225a6aa78785d466e7f033b38deb9c7fa17bd3836c9ad8884f27be84a.json index 5a6aa7dd69..313180bd10 100644 --- a/backend/.sqlx/query-b3c02fd225a6aa78785d466e7f033b38deb9c7fa17bd3836c9ad8884f27be84a.json +++ b/backend/.sqlx/query-b3c02fd225a6aa78785d466e7f033b38deb9c7fa17bd3836c9ad8884f27be84a.json @@ -30,7 +30,7 @@ "noop", "appdependencies", "deploymentcallback", - "singlescriptflow", + "singlestepflow", "flowscript", "flownode", "appscript", diff --git a/backend/.sqlx/query-366609f7e7fbd73ea807128b931eff2f1ab763fa630c8531f590fed2110c03d9.json b/backend/.sqlx/query-b474ae4401b3d4c95add2d3353eb66512801c20ad685e8ed544fe4b86601aaa8.json similarity index 86% rename from backend/.sqlx/query-366609f7e7fbd73ea807128b931eff2f1ab763fa630c8531f590fed2110c03d9.json rename to backend/.sqlx/query-b474ae4401b3d4c95add2d3353eb66512801c20ad685e8ed544fe4b86601aaa8.json index e9ec98136b..1341337cc5 100644 --- a/backend/.sqlx/query-366609f7e7fbd73ea807128b931eff2f1ab763fa630c8531f590fed2110c03d9.json +++ b/backend/.sqlx/query-b474ae4401b3d4c95add2d3353eb66512801c20ad685e8ed544fe4b86601aaa8.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO draft\n (workspace_id, path, value, typ)\n VALUES ($1, $2, $3::text::json, $4)\n ON CONFLICT (workspace_id, path, typ) DO UPDATE SET value = $3::text::json", + "query": "INSERT INTO draft\n (workspace_id, path, value, typ)\n VALUES ($1, $2, $3::text::json, $4)\n ON CONFLICT (workspace_id, path, typ) DO UPDATE SET value = EXCLUDED.value", "describe": { "columns": [], "parameters": { @@ -24,5 +24,5 @@ }, "nullable": [] }, - "hash": "366609f7e7fbd73ea807128b931eff2f1ab763fa630c8531f590fed2110c03d9" + "hash": "b474ae4401b3d4c95add2d3353eb66512801c20ad685e8ed544fe4b86601aaa8" } diff --git a/backend/.sqlx/query-f07a705df1a988827e099d146f5308b763293a27adf30d02df605317791d8126.json b/backend/.sqlx/query-b5ade857a358f2fee4bb7d005e5fef1cabea003419c891f8b1e52bc2c0156b0b.json similarity index 76% rename from backend/.sqlx/query-f07a705df1a988827e099d146f5308b763293a27adf30d02df605317791d8126.json rename to backend/.sqlx/query-b5ade857a358f2fee4bb7d005e5fef1cabea003419c891f8b1e52bc2c0156b0b.json index 0153bbf579..7857ca1b4e 100644 --- a/backend/.sqlx/query-f07a705df1a988827e099d146f5308b763293a27adf30d02df605317791d8126.json +++ b/backend/.sqlx/query-b5ade857a358f2fee4bb7d005e5fef1cabea003419c891f8b1e52bc2c0156b0b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT $1 OFFSET $2", + "query": "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only, first_time_user FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT $1 OFFSET $2", "describe": { "columns": [ { @@ -47,6 +47,11 @@ "ordinal": 8, "name": "operator_only", "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "first_time_user", + "type_info": "Bool" } ], "parameters": { @@ -64,8 +69,9 @@ true, true, true, - null + null, + false ] }, - "hash": "f07a705df1a988827e099d146f5308b763293a27adf30d02df605317791d8126" + "hash": "b5ade857a358f2fee4bb7d005e5fef1cabea003419c891f8b1e52bc2c0156b0b" } diff --git a/backend/.sqlx/query-35b211d19e53da4b64b0bd097284de3236ab939e47a1fc2b15ffc9607b552f8d.json b/backend/.sqlx/query-b5f6870444fc97d8beab3cf61c91e58936138d80a97b423c17338ba069b6a3aa.json similarity index 67% rename from backend/.sqlx/query-35b211d19e53da4b64b0bd097284de3236ab939e47a1fc2b15ffc9607b552f8d.json rename to backend/.sqlx/query-b5f6870444fc97d8beab3cf61c91e58936138d80a97b423c17338ba069b6a3aa.json index aa52f752ae..1d8fa42e31 100644 --- a/backend/.sqlx/query-35b211d19e53da4b64b0bd097284de3236ab939e47a1fc2b15ffc9607b552f8d.json +++ b/backend/.sqlx/query-b5f6870444fc97d8beab3cf61c91e58936138d80a97b423c17338ba069b6a3aa.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE \n websocket_trigger\n SET\n url = $1,\n script_path = $2,\n path = $3,\n is_flow = $4,\n filters = $5,\n initial_messages = $6,\n url_runnable_args = $7,\n edited_by = $8,\n email = $9,\n can_return_message = $10,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $13,\n error_handler_args = $14,\n retry = $15\n WHERE\n workspace_id = $11 AND path = $12\n ", + "query": "\n UPDATE \n websocket_trigger\n SET\n url = $1,\n script_path = $2,\n path = $3,\n is_flow = $4,\n filters = $5,\n initial_messages = $6,\n url_runnable_args = $7,\n edited_by = $8,\n email = $9,\n can_return_message = $10,\n can_return_error_result = $11,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE\n workspace_id = $12 AND path = $13\n ", "describe": { "columns": [], "parameters": { @@ -15,6 +15,7 @@ "Varchar", "Varchar", "Bool", + "Bool", "Text", "Text", "Varchar", @@ -24,5 +25,5 @@ }, "nullable": [] }, - "hash": "35b211d19e53da4b64b0bd097284de3236ab939e47a1fc2b15ffc9607b552f8d" + "hash": "b5f6870444fc97d8beab3cf61c91e58936138d80a97b423c17338ba069b6a3aa" } diff --git a/backend/.sqlx/query-b5fbd7893950610f1285662df24f438c9855ba860e23befd88c2544ef86e9133.json b/backend/.sqlx/query-b5fbd7893950610f1285662df24f438c9855ba860e23befd88c2544ef86e9133.json deleted file mode 100644 index 766748fa60..0000000000 --- a/backend/.sqlx/query-b5fbd7893950610f1285662df24f438c9855ba860e23befd88c2544ef86e9133.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, $4, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "b5fbd7893950610f1285662df24f438c9855ba860e23befd88c2544ef86e9133" -} diff --git a/backend/.sqlx/query-b731208c3e98f8fb15bf0c72e2ab1d4c61566c3cedf31596efe084ef7dc9985a.json b/backend/.sqlx/query-b731208c3e98f8fb15bf0c72e2ab1d4c61566c3cedf31596efe084ef7dc9985a.json new file mode 100644 index 0000000000..04b960b81a --- /dev/null +++ b/backend/.sqlx/query-b731208c3e98f8fb15bf0c72e2ab1d4c61566c3cedf31596efe084ef7dc9985a.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, slack_command_script, slack_email FROM workspace_settings WHERE slack_team_id = $1 AND slack_command_script IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "slack_command_script", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "slack_email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "b731208c3e98f8fb15bf0c72e2ab1d4c61566c3cedf31596efe084ef7dc9985a" +} diff --git a/backend/.sqlx/query-bb46e5dcf5490ef3511faa131ad5693dedf34366e51044ddf30695995d194090.json b/backend/.sqlx/query-bb46e5dcf5490ef3511faa131ad5693dedf34366e51044ddf30695995d194090.json new file mode 100644 index 0000000000..d9ca7cfa52 --- /dev/null +++ b/backend/.sqlx/query-bb46e5dcf5490ef3511faa131ad5693dedf34366e51044ddf30695995d194090.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM job_result_stream_v2 WHERE job_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "bb46e5dcf5490ef3511faa131ad5693dedf34366e51044ddf30695995d194090" +} diff --git a/backend/.sqlx/query-bbd51f4f0a8bb2db5d6b634f2d32f5f7f7f57390a0e4b6e7cad45c9147fd6356.json b/backend/.sqlx/query-bbd51f4f0a8bb2db5d6b634f2d32f5f7f7f57390a0e4b6e7cad45c9147fd6356.json new file mode 100644 index 0000000000..ab4e5e5d36 --- /dev/null +++ b/backend/.sqlx/query-bbd51f4f0a8bb2db5d6b634f2d32f5f7f7f57390a0e4b6e7cad45c9147fd6356.json @@ -0,0 +1,72 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n script_path, \n is_flow, \n workspace_id, \n edited_by, \n email, \n path, \n error_handler_path as \"error_handler_path: _\", \n error_handler_args as \"error_handler_args: _\", \n retry as \"retry: _\" \n FROM email_trigger \n WHERE workspace_id = $1 \n AND local_part = $2 \n AND (workspaced_local_part = TRUE OR $3 IS TRUE)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "error_handler_path: _", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "error_handler_args: _", + "type_info": "Jsonb" + }, + { + "ordinal": 8, + "name": "retry: _", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + true, + true + ] + }, + "hash": "bbd51f4f0a8bb2db5d6b634f2d32f5f7f7f57390a0e4b6e7cad45c9147fd6356" +} diff --git a/backend/.sqlx/query-bc9a17567cc71f51a8fa0a6fe12c0aa3a52e2ba194d40c66993d02ae2c129327.json b/backend/.sqlx/query-bc9a17567cc71f51a8fa0a6fe12c0aa3a52e2ba194d40c66993d02ae2c129327.json new file mode 100644 index 0000000000..6a5c49f43a --- /dev/null +++ b/backend/.sqlx/query-bc9a17567cc71f51a8fa0a6fe12c0aa3a52e2ba194d40c66993d02ae2c129327.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE schedule SET dynamic_skip = $1 WHERE dynamic_skip = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bc9a17567cc71f51a8fa0a6fe12c0aa3a52e2ba194d40c66993d02ae2c129327" +} diff --git a/backend/.sqlx/query-bcc4f786fa2771a4a265aec8d1c403e660002503aad860caa12ebee2dbab3f0f.json b/backend/.sqlx/query-bcc4f786fa2771a4a265aec8d1c403e660002503aad860caa12ebee2dbab3f0f.json deleted file mode 100644 index 5839ea94b4..0000000000 --- a/backend/.sqlx/query-bcc4f786fa2771a4a265aec8d1c403e660002503aad860caa12ebee2dbab3f0f.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n workspace_id,\n path,\n nats_resource_path,\n subjects,\n stream_name,\n consumer_name,\n use_jetstream,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM nats_trigger\n WHERE enabled IS TRUE AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "nats_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "subjects", - "type_info": "VarcharArray" - }, - { - "ordinal": 4, - "name": "stream_name", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "consumer_name", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "use_jetstream", - "type_info": "Bool" - }, - { - "ordinal": 7, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 9, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 10, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 12, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 13, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 14, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 15, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 16, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 17, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 18, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 19, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - true, - true, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - true, - true, - true - ] - }, - "hash": "bcc4f786fa2771a4a265aec8d1c403e660002503aad860caa12ebee2dbab3f0f" -} diff --git a/backend/.sqlx/query-bd1b62a6435cfa7d8235bcb3cb104ef01831f400ba9bd1c3433412770d1d2a82.json b/backend/.sqlx/query-bd1b62a6435cfa7d8235bcb3cb104ef01831f400ba9bd1c3433412770d1d2a82.json new file mode 100644 index 0000000000..c20d112d9e --- /dev/null +++ b/backend/.sqlx/query-bd1b62a6435cfa7d8235bcb3cb104ef01831f400ba9bd1c3433412770d1d2a82.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM autoscaling_event WHERE applied_at <= now() - ($1::bigint::text || ' s')::interval", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "bd1b62a6435cfa7d8235bcb3cb104ef01831f400ba9bd1c3433412770d1d2a82" +} diff --git a/backend/.sqlx/query-bd5d39d1ef26ac0526a2ae834b45bcf902d143eec8faec306c651e74ad14c68e.json b/backend/.sqlx/query-bd5d39d1ef26ac0526a2ae834b45bcf902d143eec8faec306c651e74ad14c68e.json new file mode 100644 index 0000000000..9fb02ee9e8 --- /dev/null +++ b/backend/.sqlx/query-bd5d39d1ef26ac0526a2ae834b45bcf902d143eec8faec306c651e74ad14c68e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_key (workspace_id, kind, key)\n SELECT $2, kind, key FROM workspace_key WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "bd5d39d1ef26ac0526a2ae834b45bcf902d143eec8faec306c651e74ad14c68e" +} diff --git a/backend/.sqlx/query-bdb1ece5c233f242cf341c089a2f2b785dfa5cc14d9be224c0707e10247ed8b7.json b/backend/.sqlx/query-bdb1ece5c233f242cf341c089a2f2b785dfa5cc14d9be224c0707e10247ed8b7.json new file mode 100644 index 0000000000..c31eba0110 --- /dev/null +++ b/backend/.sqlx/query-bdb1ece5c233f242cf341c089a2f2b785dfa5cc14d9be224c0707e10247ed8b7.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE flow_conversation SET updated_at = NOW() WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "bdb1ece5c233f242cf341c089a2f2b785dfa5cc14d9be224c0707e10247ed8b7" +} diff --git a/backend/.sqlx/query-193d292c5ed44bf5266ad52c83704c3a36aa284fab3b7e638dbca12ac846b82b.json b/backend/.sqlx/query-bf2163c542fb8c4e173167a8f333ef762fecf782424c5b61b89f32918b8d6971.json similarity index 82% rename from backend/.sqlx/query-193d292c5ed44bf5266ad52c83704c3a36aa284fab3b7e638dbca12ac846b82b.json rename to backend/.sqlx/query-bf2163c542fb8c4e173167a8f333ef762fecf782424c5b61b89f32918b8d6971.json index 2d4df84899..46db07e1e0 100644 --- a/backend/.sqlx/query-193d292c5ed44bf5266ad52c83704c3a36aa284fab3b7e638dbca12ac846b82b.json +++ b/backend/.sqlx/query-bf2163c542fb8c4e173167a8f333ef762fecf782424c5b61b89f32918b8d6971.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH inserted_job AS (\n INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job,\n created_by, permissioned_as, runnable_id, runnable_path, args, kind, trigger,\n script_lang, same_worker, pre_run_error, permissioned_as_email, visible_to_owner,\n flow_innermost_root_job, root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id,\n cache_ttl, priority, trigger_kind, script_entrypoint_override, preprocessed)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,\n $19, $20, $38, $21, $22, $23, $24, $25, $26, $39::job_trigger_kind,\n ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27)\n ),\n inserted_runtime AS (\n INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)\n ),\n inserted_job_perms AS (\n INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n values ($1, $32, $33, $34, $35, $36, $37, $2) \n ON CONFLICT (job_id) DO UPDATE SET email = $32, username = $33, is_admin = $34, is_operator = $35, folders = $36, groups = $37, workspace_id = $2\n )\n INSERT INTO v2_job_queue\n (workspace_id, id, running, scheduled_for, started_at, tag, priority)\n VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 OR $40 THEN now() END, $30, $31)", + "query": "WITH inserted_job AS (\n INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job,\n created_by, permissioned_as, runnable_id, runnable_path, args, kind, trigger,\n script_lang, same_worker, pre_run_error, permissioned_as_email, visible_to_owner,\n flow_innermost_root_job, root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id,\n cache_ttl, priority, trigger_kind, script_entrypoint_override, preprocessed)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,\n $19, $20, $38, $21, $22, $23, $24, $25, $26, $39::job_trigger_kind,\n ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27)\n ),\n inserted_runtime AS (\n INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)\n ),\n inserted_job_perms AS (\n INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, end_user_email) \n values ($1, $32, $33, $34, $35, $36, $37, $2, $41) \n ON CONFLICT (job_id) DO UPDATE SET email = EXCLUDED.email, username = EXCLUDED.username, is_admin = EXCLUDED.is_admin, is_operator = EXCLUDED.is_operator, folders = EXCLUDED.folders, groups = EXCLUDED.groups, workspace_id = EXCLUDED.workspace_id, end_user_email = EXCLUDED.end_user_email\n )\n INSERT INTO v2_job_queue\n (workspace_id, id, running, scheduled_for, started_at, tag, priority)\n VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 OR $40 THEN now() END, $30, $31)", "describe": { "columns": [], "parameters": { @@ -36,7 +36,7 @@ "noop", "appdependencies", "deploymentcallback", - "singlescriptflow", + "singlestepflow", "flowscript", "flownode", "appscript", @@ -123,10 +123,11 @@ } } }, - "Bool" + "Bool", + "Varchar" ] }, "nullable": [] }, - "hash": "193d292c5ed44bf5266ad52c83704c3a36aa284fab3b7e638dbca12ac846b82b" + "hash": "bf2163c542fb8c4e173167a8f333ef762fecf782424c5b61b89f32918b8d6971" } diff --git a/backend/.sqlx/query-bf252ea52aeb57664ad5054b39998a5e50bc98a8360387bd4158f9bc1289319f.json b/backend/.sqlx/query-bf252ea52aeb57664ad5054b39998a5e50bc98a8360387bd4158f9bc1289319f.json deleted file mode 100644 index 97929c71e3..0000000000 --- a/backend/.sqlx/query-bf252ea52aeb57664ad5054b39998a5e50bc98a8360387bd4158f9bc1289319f.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO flow_version\n (workspace_id, path, value, schema, created_by)\n\n SELECT workspace_id, path, value, schema, created_by\n FROM flow_version WHERE path = $1 AND workspace_id = $2 AND id = $3\n\n RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "bf252ea52aeb57664ad5054b39998a5e50bc98a8360387bd4158f9bc1289319f" -} diff --git a/backend/.sqlx/query-bfc534d87d701d7ac78cc97d0054d829165ba3f22fba75c3161e4cddb72264ee.json b/backend/.sqlx/query-bfc534d87d701d7ac78cc97d0054d829165ba3f22fba75c3161e4cddb72264ee.json deleted file mode 100644 index 76f0f1486f..0000000000 --- a/backend/.sqlx/query-bfc534d87d701d7ac78cc97d0054d829165ba3f22fba75c3161e4cddb72264ee.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n postgres_trigger \n SET \n enabled = FALSE, \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "bfc534d87d701d7ac78cc97d0054d829165ba3f22fba75c3161e4cddb72264ee" -} diff --git a/backend/.sqlx/query-d0037961e8e787c4277afc3eb79f3b72e3323f878387de8b6fa31493f1215a77.json b/backend/.sqlx/query-c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6.json similarity index 67% rename from backend/.sqlx/query-d0037961e8e787c4277afc3eb79f3b72e3323f878387de8b6fa31493f1215a77.json rename to backend/.sqlx/query-c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6.json index d284956ca1..bb80e8d19a 100644 --- a/backend/.sqlx/query-d0037961e8e787c4277afc3eb79f3b72e3323f878387de8b6fa31493f1215a77.json +++ b/backend/.sqlx/query-c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false", + "query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings,\n usr.disabled\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false", "describe": { "columns": [ { @@ -32,6 +32,11 @@ "ordinal": 5, "name": "operator_settings", "type_info": "Jsonb" + }, + { + "ordinal": 6, + "name": "disabled", + "type_info": "Bool" } ], "parameters": { @@ -45,8 +50,9 @@ false, true, true, - null + null, + false ] }, - "hash": "d0037961e8e787c4277afc3eb79f3b72e3323f878387de8b6fa31493f1215a77" + "hash": "c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6" } diff --git a/backend/.sqlx/query-c0e6dbce7a401b06e1bf45155c3f81572818177a6024e743b12f558b46edf74c.json b/backend/.sqlx/query-c0e6dbce7a401b06e1bf45155c3f81572818177a6024e743b12f558b46edf74c.json deleted file mode 100644 index e65e3cb60e..0000000000 --- a/backend/.sqlx/query-c0e6dbce7a401b06e1bf45155c3f81572818177a6024e743b12f558b46edf74c.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n path,\n is_flow,\n workspace_id,\n owner,\n email,\n trigger_config as \"trigger_config!: _\"\n FROM\n capture_config\n WHERE\n trigger_kind = 'gcp' AND\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n trigger_config IS NOT NULL AND\n trigger_config->>'delivery_type' IS DISTINCT FROM 'push' AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "owner", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "trigger_config!: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - false, - true - ] - }, - "hash": "c0e6dbce7a401b06e1bf45155c3f81572818177a6024e743b12f558b46edf74c" -} diff --git a/backend/.sqlx/query-c13d40a1c8db041137e0990937a65d93abfbd68b6d21c1382895238b23dd78dd.json b/backend/.sqlx/query-c13d40a1c8db041137e0990937a65d93abfbd68b6d21c1382895238b23dd78dd.json new file mode 100644 index 0000000000..32a06ab680 --- /dev/null +++ b/backend/.sqlx/query-c13d40a1c8db041137e0990937a65d93abfbd68b6d21c1382895238b23dd78dd.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT usage FROM usage\n WHERE id = $1\n AND is_workspace = TRUE\n AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "usage", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c13d40a1c8db041137e0990937a65d93abfbd68b6d21c1382895238b23dd78dd" +} diff --git a/backend/.sqlx/query-23fb2099fe211c9c5388f28097ed8635198a4144a3415121800f5df52b2a133f.json b/backend/.sqlx/query-c383cc023714b361d10c10e8fef1fc148ab1da942951ee9ffdddaecee76a6be9.json similarity index 50% rename from backend/.sqlx/query-23fb2099fe211c9c5388f28097ed8635198a4144a3415121800f5df52b2a133f.json rename to backend/.sqlx/query-c383cc023714b361d10c10e8fef1fc148ab1da942951ee9ffdddaecee76a6be9.json index 629531be93..56a3642faa 100644 --- a/backend/.sqlx/query-23fb2099fe211c9c5388f28097ed8635198a4144a3415121800f5df52b2a133f.json +++ b/backend/.sqlx/query-c383cc023714b361d10c10e8fef1fc148ab1da942951ee9ffdddaecee76a6be9.json @@ -1,54 +1,59 @@ { "db_name": "PostgreSQL", - "query": "SELECT script_path, is_flow, workspace_id, edited_by, email, path FROM email_trigger WHERE workspace_id = $1 AND local_part = $2 AND (workspaced_local_part = TRUE OR $3 IS TRUE)", + "query": "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by\n FROM flow_conversation\n WHERE id = $1 AND workspace_id = $2", "describe": { "columns": [ { "ordinal": 0, - "name": "script_path", - "type_info": "Varchar" + "name": "id", + "type_info": "Uuid" }, { "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, "name": "workspace_id", "type_info": "Varchar" }, + { + "ordinal": 2, + "name": "flow_path", + "type_info": "Varchar" + }, { "ordinal": 3, - "name": "edited_by", + "name": "title", "type_info": "Varchar" }, { "ordinal": 4, - "name": "email", - "type_info": "Varchar" + "name": "created_at", + "type_info": "Timestamptz" }, { "ordinal": 5, - "name": "path", + "name": "updated_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "created_by", "type_info": "Varchar" } ], "parameters": { "Left": [ - "Text", - "Text", - "Bool" + "Uuid", + "Text" ] }, "nullable": [ false, false, false, + true, false, false, false ] }, - "hash": "23fb2099fe211c9c5388f28097ed8635198a4144a3415121800f5df52b2a133f" + "hash": "c383cc023714b361d10c10e8fef1fc148ab1da942951ee9ffdddaecee76a6be9" } diff --git a/backend/.sqlx/query-c4121e4d1de409f66f1984ad2a962dd86568e89f278b93320911f8ae5475f038.json b/backend/.sqlx/query-c4121e4d1de409f66f1984ad2a962dd86568e89f278b93320911f8ae5475f038.json new file mode 100644 index 0000000000..a0d0a5dfcb --- /dev/null +++ b/backend/.sqlx/query-c4121e4d1de409f66f1984ad2a962dd86568e89f278b93320911f8ae5475f038.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at)\n SELECT $2, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at\n FROM variable\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "c4121e4d1de409f66f1984ad2a962dd86568e89f278b93320911f8ae5475f038" +} diff --git a/backend/.sqlx/query-ac1fd12e9ec92022be38aee0e91e9002e0e810d0e76dce5ced1000a7cb514adb.json b/backend/.sqlx/query-c444549636a8520c3ad1164491a495372104786fa968e8b7e9e8a5b8fe775f99.json similarity index 77% rename from backend/.sqlx/query-ac1fd12e9ec92022be38aee0e91e9002e0e810d0e76dce5ced1000a7cb514adb.json rename to backend/.sqlx/query-c444549636a8520c3ad1164491a495372104786fa968e8b7e9e8a5b8fe775f99.json index 420d3b8573..aa77f61309 100644 --- a/backend/.sqlx/query-ac1fd12e9ec92022be38aee0e91e9002e0e810d0e76dce5ced1000a7cb514adb.json +++ b/backend/.sqlx/query-c444549636a8520c3ad1164491a495372104786fa968e8b7e9e8a5b8fe775f99.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n workspace_id,\n slack_team_id,\n teams_team_id,\n teams_team_name,\n slack_name,\n slack_command_script,\n teams_command_script,\n slack_email,\n auto_invite_domain,\n auto_invite_operator,\n auto_add,\n customer_id,\n plan,\n webhook,\n deploy_to,\n ai_config,\n error_handler,\n error_handler_extra_args,\n error_handler_muted_on_cancel,\n large_file_storage,\n ducklake,\n git_sync,\n deploy_ui,\n default_app,\n default_scripts,\n mute_critical_alerts,\n color,\n operator_settings,\n git_app_installations,\n auto_add_instance_groups,\n auto_add_instance_groups_roles\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ", + "query": "\n SELECT\n workspace_id,\n slack_team_id,\n teams_team_id,\n teams_team_name,\n slack_name,\n slack_command_script,\n teams_command_script,\n slack_email,\n slack_oauth_client_id,\n slack_oauth_client_secret,\n auto_invite_domain,\n auto_invite_operator,\n auto_add,\n customer_id,\n plan,\n webhook,\n deploy_to,\n ai_config,\n error_handler,\n error_handler_extra_args,\n error_handler_muted_on_cancel,\n large_file_storage,\n ducklake,\n git_sync,\n deploy_ui,\n default_app,\n default_scripts,\n mute_critical_alerts,\n color,\n operator_settings,\n git_app_installations,\n auto_add_instance_groups,\n auto_add_instance_groups_roles\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ", "describe": { "columns": [ { @@ -45,116 +45,126 @@ }, { "ordinal": 8, - "name": "auto_invite_domain", + "name": "slack_oauth_client_id", "type_info": "Varchar" }, { "ordinal": 9, + "name": "slack_oauth_client_secret", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "auto_invite_domain", + "type_info": "Varchar" + }, + { + "ordinal": 11, "name": "auto_invite_operator", "type_info": "Bool" }, { - "ordinal": 10, + "ordinal": 12, "name": "auto_add", "type_info": "Bool" }, { - "ordinal": 11, + "ordinal": 13, "name": "customer_id", "type_info": "Varchar" }, { - "ordinal": 12, + "ordinal": 14, "name": "plan", "type_info": "Varchar" }, { - "ordinal": 13, + "ordinal": 15, "name": "webhook", "type_info": "Text" }, { - "ordinal": 14, + "ordinal": 16, "name": "deploy_to", "type_info": "Varchar" }, { - "ordinal": 15, + "ordinal": 17, "name": "ai_config", "type_info": "Jsonb" }, { - "ordinal": 16, + "ordinal": 18, "name": "error_handler", "type_info": "Varchar" }, { - "ordinal": 17, + "ordinal": 19, "name": "error_handler_extra_args", "type_info": "Json" }, { - "ordinal": 18, + "ordinal": 20, "name": "error_handler_muted_on_cancel", "type_info": "Bool" }, { - "ordinal": 19, + "ordinal": 21, "name": "large_file_storage", "type_info": "Jsonb" }, { - "ordinal": 20, + "ordinal": 22, "name": "ducklake", "type_info": "Jsonb" }, { - "ordinal": 21, + "ordinal": 23, "name": "git_sync", "type_info": "Jsonb" }, { - "ordinal": 22, + "ordinal": 24, "name": "deploy_ui", "type_info": "Jsonb" }, { - "ordinal": 23, + "ordinal": 25, "name": "default_app", "type_info": "Varchar" }, { - "ordinal": 24, + "ordinal": 26, "name": "default_scripts", "type_info": "Jsonb" }, { - "ordinal": 25, + "ordinal": 27, "name": "mute_critical_alerts", "type_info": "Bool" }, { - "ordinal": 26, + "ordinal": 28, "name": "color", "type_info": "Varchar" }, { - "ordinal": 27, + "ordinal": 29, "name": "operator_settings", "type_info": "Jsonb" }, { - "ordinal": 28, + "ordinal": 30, "name": "git_app_installations", "type_info": "Jsonb" }, { - "ordinal": 29, + "ordinal": 31, "name": "auto_add_instance_groups", "type_info": "TextArray" }, { - "ordinal": 30, + "ordinal": 32, "name": "auto_add_instance_groups_roles", "type_info": "Jsonb" } @@ -183,6 +193,8 @@ true, true, true, + true, + true, false, true, true, @@ -198,5 +210,5 @@ true ] }, - "hash": "ac1fd12e9ec92022be38aee0e91e9002e0e810d0e76dce5ced1000a7cb514adb" + "hash": "c444549636a8520c3ad1164491a495372104786fa968e8b7e9e8a5b8fe775f99" } diff --git a/backend/.sqlx/query-c5063e79aafa70b974276f5bea43ad71135d44b2e84c9efac43a6678f0cf9a18.json b/backend/.sqlx/query-c5063e79aafa70b974276f5bea43ad71135d44b2e84c9efac43a6678f0cf9a18.json deleted file mode 100644 index 9ab57a7e3a..0000000000 --- a/backend/.sqlx/query-c5063e79aafa70b974276f5bea43ad71135d44b2e84c9efac43a6678f0cf9a18.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n mqtt_trigger \n SET \n enabled = FALSE, \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "c5063e79aafa70b974276f5bea43ad71135d44b2e84c9efac43a6678f0cf9a18" -} diff --git a/backend/.sqlx/query-c67e81985093ff976f1326ff2254585e850f61a787a5b8f4a8d88f88016f1f2b.json b/backend/.sqlx/query-c67e81985093ff976f1326ff2254585e850f61a787a5b8f4a8d88f88016f1f2b.json new file mode 100644 index 0000000000..04e253ce54 --- /dev/null +++ b/backend/.sqlx/query-c67e81985093ff976f1326ff2254585e850f61a787a5b8f4a8d88f88016f1f2b.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_at FROM flow_conversation_message WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c67e81985093ff976f1326ff2254585e850f61a787a5b8f4a8d88f88016f1f2b" +} diff --git a/backend/.sqlx/query-9f4811fe735d401b62f4b7bf3db2b5cd13eb3364a8a8546007dec7ab528b1f9d.json b/backend/.sqlx/query-c6ef0acdf20bd71dd26de981fb49f178ba8a1b8c1e01e0fec1dfd6a54ea7a894.json similarity index 50% rename from backend/.sqlx/query-9f4811fe735d401b62f4b7bf3db2b5cd13eb3364a8a8546007dec7ab528b1f9d.json rename to backend/.sqlx/query-c6ef0acdf20bd71dd26de981fb49f178ba8a1b8c1e01e0fec1dfd6a54ea7a894.json index 4c9693ce27..0bd82635da 100644 --- a/backend/.sqlx/query-9f4811fe735d401b62f4b7bf3db2b5cd13eb3364a8a8546007dec7ab528b1f9d.json +++ b/backend/.sqlx/query-c6ef0acdf20bd71dd26de981fb49f178ba8a1b8c1e01e0fec1dfd6a54ea7a894.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\nWITH rows_to_delete AS (\n SELECT concurrency_id\n FROM concurrency_counter\n WHERE job_uuids = '{}'::jsonb\n FOR UPDATE SKIP LOCKED\n)\nDELETE FROM concurrency_counter\nWHERE concurrency_id IN (SELECT concurrency_id FROM rows_to_delete) RETURNING concurrency_id", + "query": "\nWITH rows_to_delete AS (\n SELECT concurrency_id\n FROM concurrency_counter\n \n WHERE job_uuids = '{}'::jsonb\n FOR UPDATE SKIP LOCKED\n)\nDELETE FROM concurrency_counter\nWHERE concurrency_id IN (SELECT concurrency_id FROM rows_to_delete) RETURNING concurrency_id", "describe": { "columns": [ { @@ -16,5 +16,5 @@ false ] }, - "hash": "9f4811fe735d401b62f4b7bf3db2b5cd13eb3364a8a8546007dec7ab528b1f9d" + "hash": "c6ef0acdf20bd71dd26de981fb49f178ba8a1b8c1e01e0fec1dfd6a54ea7a894" } diff --git a/backend/.sqlx/query-cba3bfb174829ee3b08ea195831fdba4335ca221a5fd25c72519ac376c593e44.json b/backend/.sqlx/query-cba3bfb174829ee3b08ea195831fdba4335ca221a5fd25c72519ac376c593e44.json deleted file mode 100644 index ebda0390f9..0000000000 --- a/backend/.sqlx/query-cba3bfb174829ee3b08ea195831fdba4335ca221a5fd25c72519ac376c593e44.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = 'websocket' AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "cba3bfb174829ee3b08ea195831fdba4335ca221a5fd25c72519ac376c593e44" -} diff --git a/backend/.sqlx/query-cbb93da0b7719a27d2ae1ec0f653322cae965dc5d8ebc98f69d5922fa1192561.json b/backend/.sqlx/query-cbb93da0b7719a27d2ae1ec0f653322cae965dc5d8ebc98f69d5922fa1192561.json deleted file mode 100644 index 684e857290..0000000000 --- a/backend/.sqlx/query-cbb93da0b7719a27d2ae1ec0f653322cae965dc5d8ebc98f69d5922fa1192561.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT COALESCE(\n (SELECT COUNT(*) FROM jsonb_object_keys(job_uuids)),\n 0\n )\n FROM concurrency_counter \n WHERE concurrency_id = $1\n FOR UPDATE", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "coalesce", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "cbb93da0b7719a27d2ae1ec0f653322cae965dc5d8ebc98f69d5922fa1192561" -} diff --git a/backend/.sqlx/query-afc7c23c057748f6d4a61dbef17e433b8875c6588b91e38e8141d12189118412.json b/backend/.sqlx/query-cc8e10a4f39e118b145cc8f6fbd5f3db1fb30c9564ed039ad4c952333ec29b39.json similarity index 74% rename from backend/.sqlx/query-afc7c23c057748f6d4a61dbef17e433b8875c6588b91e38e8141d12189118412.json rename to backend/.sqlx/query-cc8e10a4f39e118b145cc8f6fbd5f3db1fb30c9564ed039ad4c952333ec29b39.json index 08ea345138..49df64b555 100644 --- a/backend/.sqlx/query-afc7c23c057748f6d4a61dbef17e433b8875c6588b91e38e8141d12189118412.json +++ b/backend/.sqlx/query-cc8e10a4f39e118b145cc8f6fbd5f3db1fb30c9564ed039ad4c952333ec29b39.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO deployment_metadata (workspace_id, path, flow_version, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL DO UPDATE SET deployment_msg = $4", + "query": "INSERT INTO deployment_metadata (workspace_id, path, flow_version, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL DO UPDATE SET deployment_msg = EXCLUDED.deployment_msg", "describe": { "columns": [], "parameters": { @@ -13,5 +13,5 @@ }, "nullable": [] }, - "hash": "afc7c23c057748f6d4a61dbef17e433b8875c6588b91e38e8141d12189118412" + "hash": "cc8e10a4f39e118b145cc8f6fbd5f3db1fb30c9564ed039ad4c952333ec29b39" } diff --git a/backend/.sqlx/query-cd33a9d63f4706a7e3b1e23cd0a4b2e3ecb30aae6510d1fcd08493b07c8b0952.json b/backend/.sqlx/query-cd33a9d63f4706a7e3b1e23cd0a4b2e3ecb30aae6510d1fcd08493b07c8b0952.json deleted file mode 100644 index 36b1323980..0000000000 --- a/backend/.sqlx/query-cd33a9d63f4706a7e3b1e23cd0a4b2e3ecb30aae6510d1fcd08493b07c8b0952.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE websocket_trigger SET enabled = FALSE, error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "cd33a9d63f4706a7e3b1e23cd0a4b2e3ecb30aae6510d1fcd08493b07c8b0952" -} diff --git a/backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json b/backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json deleted file mode 100644 index 912b9a53de..0000000000 --- a/backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE nats_trigger SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE enabled IS TRUE AND workspace_id = $2 AND path = $3 AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674" -} diff --git a/backend/.sqlx/query-cf76aedfdffc25057a64fc94fd6be0791d5970458810a2b465c27f02ad1d4b2f.json b/backend/.sqlx/query-cf76aedfdffc25057a64fc94fd6be0791d5970458810a2b465c27f02ad1d4b2f.json deleted file mode 100644 index 6557fadd07..0000000000 --- a/backend/.sqlx/query-cf76aedfdffc25057a64fc94fd6be0791d5970458810a2b465c27f02ad1d4b2f.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n workspace_id,\n path,\n script_path,\n replication_slot_name,\n publication_name,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n postgres_resource_path,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM\n postgres_trigger\n WHERE\n enabled IS TRUE\n AND (last_server_ping IS NULL OR\n last_server_ping < now() - interval '15 seconds'\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "replication_slot_name", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "publication_name", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 6, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 9, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 10, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 11, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 12, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 13, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 14, - "name": "postgres_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 15, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 16, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 17, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - true, - false, - false, - true, - true, - true - ] - }, - "hash": "cf76aedfdffc25057a64fc94fd6be0791d5970458810a2b465c27f02ad1d4b2f" -} diff --git a/backend/.sqlx/query-cfe06702916362aaf5122bb95593eff389e0d44b7a58b69fd5c79629599902fc.json b/backend/.sqlx/query-cfe06702916362aaf5122bb95593eff389e0d44b7a58b69fd5c79629599902fc.json new file mode 100644 index 0000000000..e9cedd467d --- /dev/null +++ b/backend/.sqlx/query-cfe06702916362aaf5122bb95593eff389e0d44b7a58b69fd5c79629599902fc.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM debounce_stale_data WHERE job_id = $1 RETURNING to_relock", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "to_relock", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "cfe06702916362aaf5122bb95593eff389e0d44b7a58b69fd5c79629599902fc" +} diff --git a/backend/.sqlx/query-cff764601318bfecb8e0f15b64ff9f430d7b1efe8ba863cbcc4a49bab4951794.json b/backend/.sqlx/query-cff764601318bfecb8e0f15b64ff9f430d7b1efe8ba863cbcc4a49bab4951794.json deleted file mode 100644 index a9e508d01d..0000000000 --- a/backend/.sqlx/query-cff764601318bfecb8e0f15b64ff9f430d7b1efe8ba863cbcc4a49bab4951794.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n last_server_ping = now(), \n error = $1 \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'mqtt' AND \n server_id = $5 AND \n last_client_ping > NOW() - INTERVAL '10 seconds' \n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "cff764601318bfecb8e0f15b64ff9f430d7b1efe8ba863cbcc4a49bab4951794" -} diff --git a/backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json b/backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json deleted file mode 100644 index 79fe0873dd..0000000000 --- a/backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE last_client_ping > NOW() - INTERVAL '10 seconds' AND workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'kafka' AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3" -} diff --git a/backend/.sqlx/query-d08f34000c3d96ccd0f44ca8520f966d751a4dda554d8215eedb8f65be98e100.json b/backend/.sqlx/query-d08f34000c3d96ccd0f44ca8520f966d751a4dda554d8215eedb8f65be98e100.json deleted file mode 100644 index 431ffc04a2..0000000000 --- a/backend/.sqlx/query-d08f34000c3d96ccd0f44ca8520f966d751a4dda554d8215eedb8f65be98e100.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket'", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "d08f34000c3d96ccd0f44ca8520f966d751a4dda554d8215eedb8f65be98e100" -} diff --git a/backend/.sqlx/query-d142ff8b56f4b69c20815230b5b763ce8ce9fbe89abe83989f52ff3657690fef.json b/backend/.sqlx/query-d142ff8b56f4b69c20815230b5b763ce8ce9fbe89abe83989f52ff3657690fef.json new file mode 100644 index 0000000000..8d92b42219 --- /dev/null +++ b/backend/.sqlx/query-d142ff8b56f4b69c20815230b5b763ce8ce9fbe89abe83989f52ff3657690fef.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO job_result_stream_v2 (workspace_id, job_id, stream, idx)\n VALUES (\n $1, \n $2,\n $3, \n $4\n )\n ON CONFLICT (job_id, idx) DO UPDATE SET stream = job_result_stream_v2.stream || EXCLUDED.stream\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "d142ff8b56f4b69c20815230b5b763ce8ce9fbe89abe83989f52ff3657690fef" +} diff --git a/backend/.sqlx/query-d2fe10b3e608407147adb704a7cf3da60f22dcce6f05852de2f75c737b57907f.json b/backend/.sqlx/query-d2fe10b3e608407147adb704a7cf3da60f22dcce6f05852de2f75c737b57907f.json new file mode 100644 index 0000000000..8e693056a0 --- /dev/null +++ b/backend/.sqlx/query-d2fe10b3e608407147adb704a7cf3da60f22dcce6f05852de2f75c737b57907f.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n schema \n FROM \n flow \n WHERE \n workspace_id = $1 AND \n path = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "schema", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "d2fe10b3e608407147adb704a7cf3da60f22dcce6f05852de2f75c737b57907f" +} diff --git a/backend/.sqlx/query-d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd.json b/backend/.sqlx/query-d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd.json new file mode 100644 index 0000000000..73fb2e98d8 --- /dev/null +++ b/backend/.sqlx/query-d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd.json @@ -0,0 +1,239 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, q.workspace_id, j.runnable_id as \"runnable_id: ScriptHash\", scheduled_for, parent_job, flow_innermost_root_job, runnable_path, kind as \"kind: JobKind\", started_at, permissioned_as, created_by, script_lang as \"script_lang: ScriptLang\", \n permissioned_as_email, flow_step_id, trigger_kind as \"trigger_kind: JobTriggerKind\", trigger, q.priority, concurrent_limit, q.tag, cache_ttl, r.ping as last_ping, worker, memory_peak, running\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE j.id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "runnable_id: ScriptHash", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 5, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 6, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "kind: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent" + ] + } + } + } + }, + { + "ordinal": 8, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "script_lang: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + }, + { + "ordinal": 12, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "trigger_kind: JobTriggerKind", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt" + ] + } + } + } + }, + { + "ordinal": 15, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 17, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 18, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 19, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 20, + "name": "last_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 21, + "name": "worker", + "type_info": "Varchar" + }, + { + "ordinal": 22, + "name": "memory_peak", + "type_info": "Int4" + }, + { + "ordinal": 23, + "name": "running", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + false, + true, + true, + true, + false, + true, + false, + false, + true, + false, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + false + ] + }, + "hash": "d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd" +} diff --git a/backend/.sqlx/query-16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f.json b/backend/.sqlx/query-d5661c7557cf3a8dee7cf799cd364d21d38edb827d2c08b0ca7d72311b78d574.json similarity index 71% rename from backend/.sqlx/query-16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f.json rename to backend/.sqlx/query-d5661c7557cf3a8dee7cf799cd364d21d38edb827d2c08b0ca7d72311b78d574.json index 1af42ff529..907b140fdd 100644 --- a/backend/.sqlx/query-16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f.json +++ b/backend/.sqlx/query-d5661c7557cf3a8dee7cf799cd364d21d38edb827d2c08b0ca7d72311b78d574.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2", + "query": "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false ORDER BY created_at DESC LIMIT 1", "describe": { "columns": [ { @@ -19,5 +19,5 @@ false ] }, - "hash": "16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f" + "hash": "d5661c7557cf3a8dee7cf799cd364d21d38edb827d2c08b0ca7d72311b78d574" } diff --git a/backend/.sqlx/query-d5d97005eebcb2760216dbd10872acdb42868fd5f7a27f71836e7a6ff1c69856.json b/backend/.sqlx/query-d5d97005eebcb2760216dbd10872acdb42868fd5f7a27f71836e7a6ff1c69856.json deleted file mode 100644 index 87361665b5..0000000000 --- a/backend/.sqlx/query-d5d97005eebcb2760216dbd10872acdb42868fd5f7a27f71836e7a6ff1c69856.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "d5d97005eebcb2760216dbd10872acdb42868fd5f7a27f71836e7a6ff1c69856" -} diff --git a/backend/.sqlx/query-d6db1103fee4bad6831656d77a28254c9a832b4f660ec755f4fa14f6f7bb3353.json b/backend/.sqlx/query-d6db1103fee4bad6831656d77a28254c9a832b4f660ec755f4fa14f6f7bb3353.json deleted file mode 100644 index 09f24968f3..0000000000 --- a/backend/.sqlx/query-d6db1103fee4bad6831656d77a28254c9a832b4f660ec755f4fa14f6f7bb3353.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n )\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid", - "Text", - "Jsonb" - ] - }, - "nullable": [ - null - ] - }, - "hash": "d6db1103fee4bad6831656d77a28254c9a832b4f660ec755f4fa14f6f7bb3353" -} diff --git a/backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json b/backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json deleted file mode 100644 index cdb8b928b3..0000000000 --- a/backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE kafka_trigger SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE enabled IS TRUE AND workspace_id = $2 AND path = $3 AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b" -} diff --git a/backend/.sqlx/query-d8c209b177da2e147a3549c888969478cd80aa157700e5c1f3b9f4d12dd31a1d.json b/backend/.sqlx/query-d8c209b177da2e147a3549c888969478cd80aa157700e5c1f3b9f4d12dd31a1d.json deleted file mode 100644 index 76c37b7a08..0000000000 --- a/backend/.sqlx/query-d8c209b177da2e147a3549c888969478cd80aa157700e5c1f3b9f4d12dd31a1d.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n COALESCE(jc.result, NULL) as \"result: sqlx::types::Json>\",\n SUBSTR(rs.stream, $3) AS \"result_stream: Option\",\n CHAR_LENGTH(rs.stream) + 1 AS stream_offset\n FROM (\n SELECT $2::uuid as job_id, $1::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "result: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 1, - "name": "result_stream: Option", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "stream_offset", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Int4" - ] - }, - "nullable": [ - null, - null, - null - ] - }, - "hash": "d8c209b177da2e147a3549c888969478cd80aa157700e5c1f3b9f4d12dd31a1d" -} diff --git a/backend/.sqlx/query-d8ea17fba0e417333e9c6cf82ad36a57830a791c8358e30e9be012b815f5e8e3.json b/backend/.sqlx/query-d8ea17fba0e417333e9c6cf82ad36a57830a791c8358e30e9be012b815f5e8e3.json deleted file mode 100644 index dcaee3cfe8..0000000000 --- a/backend/.sqlx/query-d8ea17fba0e417333e9c6cf82ad36a57830a791c8358e30e9be012b815f5e8e3.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT path, is_flow, workspace_id, trigger_config as \"trigger_config!: _\", owner, email FROM capture_config WHERE trigger_kind = 'kafka' AND last_client_ping > NOW() - INTERVAL '10 seconds' AND trigger_config IS NOT NULL AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "trigger_config!: _", - "type_info": "Jsonb" - }, - { - "ordinal": 4, - "name": "owner", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "email", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - true, - false, - false - ] - }, - "hash": "d8ea17fba0e417333e9c6cf82ad36a57830a791c8358e30e9be012b815f5e8e3" -} diff --git a/backend/.sqlx/query-d9a6f75e4c4a1f61e55b313cc09bceffac637548841897341672da427a9140fc.json b/backend/.sqlx/query-d9a6f75e4c4a1f61e55b313cc09bceffac637548841897341672da427a9140fc.json deleted file mode 100644 index 3ccfcc9582..0000000000 --- a/backend/.sqlx/query-d9a6f75e4c4a1f61e55b313cc09bceffac637548841897341672da427a9140fc.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'kafka' AND server_id = $5 AND last_client_ping > NOW() - INTERVAL '10 seconds' RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "d9a6f75e4c4a1f61e55b313cc09bceffac637548841897341672da427a9140fc" -} diff --git a/backend/.sqlx/query-fb581eb6f883ebb5909a00c65a6e1217f088290fce0e052171230bcd88f945b9.json b/backend/.sqlx/query-da7e23a32f284c9760614735d459474d2f8513202dfb64767b30e7b50833b857.json similarity index 57% rename from backend/.sqlx/query-fb581eb6f883ebb5909a00c65a6e1217f088290fce0e052171230bcd88f945b9.json rename to backend/.sqlx/query-da7e23a32f284c9760614735d459474d2f8513202dfb64767b30e7b50833b857.json index b3f640dd76..e54180983e 100644 --- a/backend/.sqlx/query-fb581eb6f883ebb5909a00c65a6e1217f088290fce0e052171230bcd88f945b9.json +++ b/backend/.sqlx/query-da7e23a32f284c9760614735d459474d2f8513202dfb64767b30e7b50833b857.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO global_settings (name, value) VALUES ('slack', $1) ON CONFLICT (name) DO UPDATE SET value = $1, updated_at = now()", + "query": "INSERT INTO global_settings (name, value) VALUES ('slack', $1) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "fb581eb6f883ebb5909a00c65a6e1217f088290fce0e052171230bcd88f945b9" + "hash": "da7e23a32f284c9760614735d459474d2f8513202dfb64767b30e7b50833b857" } diff --git a/backend/.sqlx/query-daf9674838fb3e3653a356c7434c719616a614d77e726433737e5f5d9bd60134.json b/backend/.sqlx/query-daf9674838fb3e3653a356c7434c719616a614d77e726433737e5f5d9bd60134.json deleted file mode 100644 index 2e2a9ba027..0000000000 --- a/backend/.sqlx/query-daf9674838fb3e3653a356c7434c719616a614d77e726433737e5f5d9bd60134.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH to_update AS (\n SELECT q.id, q.workspace_id, r.ping, COALESCE(zjc.counter, 0) as counter\n FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n JOIN v2_job_runtime r ON r.id = j.id\n LEFT JOIN zombie_job_counter zjc ON zjc.job_id = q.id\n WHERE ping < now() - ($1 || ' seconds')::interval\n AND running = true\n AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow')\n AND same_worker = false\n AND (zjc.counter IS NULL OR zjc.counter <= $2)\n FOR UPDATE of q SKIP LOCKED\n ),\n zombie_jobs AS (\n UPDATE v2_job_queue q\n SET running = false, started_at = null\n FROM to_update tu\n WHERE q.id = tu.id AND (tu.counter IS NULL OR tu.counter < $2)\n RETURNING q.id, q.workspace_id, ping, tu.counter\n ),\n update_ping AS (\n UPDATE v2_job_runtime r\n SET ping = null\n FROM zombie_jobs zj\n WHERE r.id = zj.id\n ),\n increment_counter AS (\n INSERT INTO zombie_job_counter (job_id, counter)\n SELECT id, 1 FROM to_update WHERE counter < $2\n ON CONFLICT (job_id) DO UPDATE\n SET counter = zombie_job_counter.counter + 1\n ),\n update_concurrency AS (\n UPDATE concurrency_counter cc\n SET job_uuids = job_uuids - zj.id::text\n FROM zombie_jobs zj\n INNER JOIN concurrency_key ck ON ck.job_id = zj.id\n WHERE cc.concurrency_id = ck.key\n )\n SELECT id AS \"id!\", workspace_id AS \"workspace_id!\", ping, counter + 1 AS counter FROM to_update", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id!", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "workspace_id!", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 3, - "name": "counter", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Int4" - ] - }, - "nullable": [ - false, - false, - true, - null - ] - }, - "hash": "daf9674838fb3e3653a356c7434c719616a614d77e726433737e5f5d9bd60134" -} diff --git a/backend/.sqlx/query-c4e0f3eab227a798d9cd7478db50a7c1a588e69c2c8d9a1def9276ba326453d2.json b/backend/.sqlx/query-db201730803047bfabccf5f10456243b590b2dd0cfe41aa5100a7e829afad9d3.json similarity index 63% rename from backend/.sqlx/query-c4e0f3eab227a798d9cd7478db50a7c1a588e69c2c8d9a1def9276ba326453d2.json rename to backend/.sqlx/query-db201730803047bfabccf5f10456243b590b2dd0cfe41aa5100a7e829afad9d3.json index bd16dab3bf..033d77862e 100644 --- a/backend/.sqlx/query-c4e0f3eab227a798d9cd7478db50a7c1a588e69c2c8d9a1def9276ba326453d2.json +++ b/backend/.sqlx/query-db201730803047bfabccf5f10456243b590b2dd0cfe41aa5100a7e829afad9d3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO cloud_workspace_settings (workspace_id, threshold_alert_amount) VALUES ($1, $2) ON CONFLICT (workspace_id) DO UPDATE SET threshold_alert_amount = $2, last_alert_sent = NULL", + "query": "INSERT INTO cloud_workspace_settings (workspace_id, threshold_alert_amount) VALUES ($1, $2) ON CONFLICT (workspace_id) DO UPDATE SET threshold_alert_amount = EXCLUDED.threshold_alert_amount, last_alert_sent = NULL", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "c4e0f3eab227a798d9cd7478db50a7c1a588e69c2c8d9a1def9276ba326453d2" + "hash": "db201730803047bfabccf5f10456243b590b2dd0cfe41aa5100a7e829afad9d3" } diff --git a/backend/.sqlx/query-db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899.json b/backend/.sqlx/query-db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899.json new file mode 100644 index 0000000000..1b48864951 --- /dev/null +++ b/backend/.sqlx/query-db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = $5\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899" +} diff --git a/backend/.sqlx/query-b7ba64475398162ba43001a9932ac9a2b9ce8295e4ac85421091ab7da17a9a7c.json b/backend/.sqlx/query-dbd66bee283a7b4f892673f968f92adce1be8b897288b655505729c97c6e324c.json similarity index 68% rename from backend/.sqlx/query-b7ba64475398162ba43001a9932ac9a2b9ce8295e4ac85421091ab7da17a9a7c.json rename to backend/.sqlx/query-dbd66bee283a7b4f892673f968f92adce1be8b897288b655505729c97c6e324c.json index 1ccac6145e..8049f4de1a 100644 --- a/backend/.sqlx/query-b7ba64475398162ba43001a9932ac9a2b9ce8295e4ac85421091ab7da17a9a7c.json +++ b/backend/.sqlx/query-dbd66bee283a7b4f892673f968f92adce1be8b897288b655505729c97c6e324c.json @@ -1,16 +1,15 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, edited_at, created_by)\n SELECT $2, name, display_name, owners, extra_perms, summary, edited_at, $3\n FROM folder\n WHERE workspace_id = $1", + "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, edited_at, created_by)\n SELECT $2, name, display_name, owners, extra_perms, summary, edited_at, created_by\n FROM folder\n WHERE workspace_id = $1", "describe": { "columns": [], "parameters": { "Left": [ "Text", - "Varchar", "Varchar" ] }, "nullable": [] }, - "hash": "b7ba64475398162ba43001a9932ac9a2b9ce8295e4ac85421091ab7da17a9a7c" + "hash": "dbd66bee283a7b4f892673f968f92adce1be8b897288b655505729c97c6e324c" } diff --git a/backend/.sqlx/query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json b/backend/.sqlx/query-dbe2025b2d7dfc985e8e5e1119fc5e5ab77c873acae357c4adaf5b82fe8f4bc5.json similarity index 69% rename from backend/.sqlx/query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json rename to backend/.sqlx/query-dbe2025b2d7dfc985e8e5e1119fc5e5ab77c873acae357c4adaf5b82fe8f4bc5.json index 1521b9fd2f..101a63dca6 100644 --- a/backend/.sqlx/query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json +++ b/backend/.sqlx/query-dbe2025b2d7dfc985e8e5e1119fc5e5ab77c873acae357c4adaf5b82fe8f4bc5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at, worker_group FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at", + "query": "SELECT worker, worker_instance, worker_group, vcpus, memory, ping_at, started_at, custom_tags FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at", "describe": { "columns": [ { @@ -15,42 +15,48 @@ }, { "ordinal": 2, + "name": "worker_group", + "type_info": "Varchar" + }, + { + "ordinal": 3, "name": "vcpus", "type_info": "Int8" }, { - "ordinal": 3, + "ordinal": 4, "name": "memory", "type_info": "Int8" }, { - "ordinal": 4, + "ordinal": 5, "name": "ping_at", "type_info": "Timestamptz" }, { - "ordinal": 5, + "ordinal": 6, "name": "started_at", "type_info": "Timestamptz" }, { - "ordinal": 6, - "name": "worker_group", - "type_info": "Varchar" + "ordinal": 7, + "name": "custom_tags", + "type_info": "TextArray" } ], "parameters": { "Left": [] }, "nullable": [ + false, false, false, true, true, false, false, - false + true ] }, - "hash": "6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973" + "hash": "dbe2025b2d7dfc985e8e5e1119fc5e5ab77c873acae357c4adaf5b82fe8f4bc5" } diff --git a/backend/.sqlx/query-dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e.json b/backend/.sqlx/query-dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e.json new file mode 100644 index 0000000000..95b23e62bf --- /dev/null +++ b/backend/.sqlx/query-dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE \n capture_config\n SET \n last_server_ping = NULL\n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = $4 AND \n server_id IS NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e" +} diff --git a/backend/.sqlx/query-dcf03ff4b922b93be37d2be5da6884ae3e8c6cf7eaa2d9c62056366eb42f2276.json b/backend/.sqlx/query-dcf03ff4b922b93be37d2be5da6884ae3e8c6cf7eaa2d9c62056366eb42f2276.json deleted file mode 100644 index 989e4612ec..0000000000 --- a/backend/.sqlx/query-dcf03ff4b922b93be37d2be5da6884ae3e8c6cf7eaa2d9c62056366eb42f2276.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE nats_trigger SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND server_id = $4 AND enabled IS TRUE RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "dcf03ff4b922b93be37d2be5da6884ae3e8c6cf7eaa2d9c62056366eb42f2276" -} diff --git a/backend/.sqlx/query-dd89d652154748d6d7e625e31778f6885d0ee62d29a4b8894a4b459dd215a103.json b/backend/.sqlx/query-dd89d652154748d6d7e625e31778f6885d0ee62d29a4b8894a4b459dd215a103.json new file mode 100644 index 0000000000..4a82795a3c --- /dev/null +++ b/backend/.sqlx/query-dd89d652154748d6d7e625e31778f6885d0ee62d29a4b8894a4b459dd215a103.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n j.args as \"args: Json>>\",\n js.flow_status as \"flow_status: Json\"\n FROM v2_job_status js\n INNER JOIN v2_job j ON j.id = js.id\n WHERE js.id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args: Json>>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "flow_status: Json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "dd89d652154748d6d7e625e31778f6885d0ee62d29a4b8894a4b459dd215a103" +} diff --git a/backend/.sqlx/query-dd92bce0ddbfbf06834807aab4d589fc104647cae7abf37ddce3ef7109726261.json b/backend/.sqlx/query-dd92bce0ddbfbf06834807aab4d589fc104647cae7abf37ddce3ef7109726261.json new file mode 100644 index 0000000000..d30b0bae62 --- /dev/null +++ b/backend/.sqlx/query-dd92bce0ddbfbf06834807aab4d589fc104647cae7abf37ddce3ef7109726261.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n string_agg(stream, '' order by idx asc) as stream, \n max(idx) + 1 as offset \n FROM job_result_stream_v2\n WHERE job_id = $2 AND idx >= $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "stream", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "offset", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "dd92bce0ddbfbf06834807aab4d589fc104647cae7abf37ddce3ef7109726261" +} diff --git a/backend/.sqlx/query-df2a426658f0a36683cc2163fe70912d0221d191b8ab091601cb53cdd932ad58.json b/backend/.sqlx/query-df2a426658f0a36683cc2163fe70912d0221d191b8ab091601cb53cdd932ad58.json deleted file mode 100644 index f98dd01404..0000000000 --- a/backend/.sqlx/query-df2a426658f0a36683cc2163fe70912d0221d191b8ab091601cb53cdd932ad58.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n gcp_trigger\n SET \n last_server_ping = now(),\n error = $1\n WHERE\n workspace_id = $2\n AND path = $3\n AND server_id = $4 \n AND enabled IS TRUE\n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "df2a426658f0a36683cc2163fe70912d0221d191b8ab091601cb53cdd932ad58" -} diff --git a/backend/.sqlx/query-e03cfb65fc9b19c7e68e43d14a6e5041db783f1e6d545533a2d95f54e616b9fb.json b/backend/.sqlx/query-e03cfb65fc9b19c7e68e43d14a6e5041db783f1e6d545533a2d95f54e616b9fb.json new file mode 100644 index 0000000000..c083a13f88 --- /dev/null +++ b/backend/.sqlx/query-e03cfb65fc9b19c7e68e43d14a6e5041db783f1e6d545533a2d95f54e616b9fb.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n error_handler \n FROM \n workspace_settings \n WHERE \n workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "error_handler", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "e03cfb65fc9b19c7e68e43d14a6e5041db783f1e6d545533a2d95f54e616b9fb" +} diff --git a/backend/.sqlx/query-9b60fa8a1003015bc5a7cdbee9a4486b313d45347dfd9d4793d60e2760763ca3.json b/backend/.sqlx/query-e1409c67b93881cf68f6ac9c8bae0856cf426c7e7860c8b5b799972baa8e6945.json similarity index 66% rename from backend/.sqlx/query-9b60fa8a1003015bc5a7cdbee9a4486b313d45347dfd9d4793d60e2760763ca3.json rename to backend/.sqlx/query-e1409c67b93881cf68f6ac9c8bae0856cf426c7e7860c8b5b799972baa8e6945.json index f4251250be..da6ca296b1 100644 --- a/backend/.sqlx/query-9b60fa8a1003015bc5a7cdbee9a4486b313d45347dfd9d4793d60e2760763ca3.json +++ b/backend/.sqlx/query-e1409c67b93881cf68f6ac9c8bae0856cf426c7e7860c8b5b799972baa8e6945.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by, flow_version.id AS version\n FROM flow\n INNER JOIN flow_version\n ON flow_version.id = $3\n WHERE flow.path = $1 and flow.workspace_id = $2", + "query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, (flow_version.value->>'chat_input_enabled')::boolean as chat_input_enabled, on_behalf_of_email, edited_by, flow_version.id AS version\n FROM flow\n INNER JOIN flow_version\n ON flow_version.id = $3\n WHERE flow.path = $1 and flow.workspace_id = $2", "describe": { "columns": [ { @@ -25,16 +25,21 @@ }, { "ordinal": 4, + "name": "chat_input_enabled", + "type_info": "Bool" + }, + { + "ordinal": 5, "name": "on_behalf_of_email", "type_info": "Text" }, { - "ordinal": 5, + "ordinal": 6, "name": "edited_by", "type_info": "Varchar" }, { - "ordinal": 6, + "ordinal": 7, "name": "version", "type_info": "Int8" } @@ -51,10 +56,11 @@ true, null, null, + null, true, false, false ] }, - "hash": "9b60fa8a1003015bc5a7cdbee9a4486b313d45347dfd9d4793d60e2760763ca3" + "hash": "e1409c67b93881cf68f6ac9c8bae0856cf426c7e7860c8b5b799972baa8e6945" } diff --git a/backend/.sqlx/query-a2e86f169ffbf8acee5f7c7b71db5859ac94ffbad267c9cd6c652e8ce8fc5d3c.json b/backend/.sqlx/query-e16f464c7e302e80f1017ff6550716448202c1181ca0b36936a9c8840c14a95d.json similarity index 76% rename from backend/.sqlx/query-a2e86f169ffbf8acee5f7c7b71db5859ac94ffbad267c9cd6c652e8ce8fc5d3c.json rename to backend/.sqlx/query-e16f464c7e302e80f1017ff6550716448202c1181ca0b36936a9c8840c14a95d.json index 4e0d53b0f3..3dc6dd5339 100644 --- a/backend/.sqlx/query-a2e86f169ffbf8acee5f7c7b71db5859ac94ffbad267c9cd6c652e8ce8fc5d3c.json +++ b/backend/.sqlx/query-e16f464c7e302e80f1017ff6550716448202c1181ca0b36936a9c8840c14a95d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, $1::text) RETURNING length(logs)", + "query": "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, EXCLUDED.logs) RETURNING length(logs)", "describe": { "columns": [ { @@ -20,5 +20,5 @@ null ] }, - "hash": "a2e86f169ffbf8acee5f7c7b71db5859ac94ffbad267c9cd6c652e8ce8fc5d3c" + "hash": "e16f464c7e302e80f1017ff6550716448202c1181ca0b36936a9c8840c14a95d" } diff --git a/backend/.sqlx/query-958ed17dafffdd37e636ccd244dc4ca60cbf562e6f6a371d5f9a9943fb30254c.json b/backend/.sqlx/query-e32d6c6ae4e0d824c4cf19128182d67f36d1fd87fe4cf4f002e18367088c497c.json similarity index 68% rename from backend/.sqlx/query-958ed17dafffdd37e636ccd244dc4ca60cbf562e6f6a371d5f9a9943fb30254c.json rename to backend/.sqlx/query-e32d6c6ae4e0d824c4cf19128182d67f36d1fd87fe4cf4f002e18367088c497c.json index 31501b4094..915cf7416b 100644 --- a/backend/.sqlx/query-958ed17dafffdd37e636ccd244dc4ca60cbf562e6f6a371d5f9a9943fb30254c.json +++ b/backend/.sqlx/query-e32d6c6ae4e0d824c4cf19128182d67f36d1fd87fe4cf4f002e18367088c497c.json @@ -1,18 +1,18 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id)\n VALUES ($1, $2, $4::text::IMPORTER_KIND, $3, $5) ON CONFLICT DO NOTHING", + "query": "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id)\n VALUES ($1, $2, $3::text::IMPORTER_KIND, $4, $5) ON CONFLICT DO NOTHING", "describe": { "columns": [], "parameters": { "Left": [ - "Varchar", "Varchar", "Varchar", "Text", + "Varchar", "Varchar" ] }, "nullable": [] }, - "hash": "958ed17dafffdd37e636ccd244dc4ca60cbf562e6f6a371d5f9a9943fb30254c" + "hash": "e32d6c6ae4e0d824c4cf19128182d67f36d1fd87fe4cf4f002e18367088c497c" } diff --git a/backend/.sqlx/query-e37ba13aa3174931f0bfcff26dbc141fe8a346ab6a1a3bc924794bfe3c9af306.json b/backend/.sqlx/query-e37ba13aa3174931f0bfcff26dbc141fe8a346ab6a1a3bc924794bfe3c9af306.json deleted file mode 100644 index cec42e51c6..0000000000 --- a/backend/.sqlx/query-e37ba13aa3174931f0bfcff26dbc141fe8a346ab6a1a3bc924794bfe3c9af306.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'nats' AND server_id = $5 AND last_client_ping > NOW() - INTERVAL '10 seconds' RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "e37ba13aa3174931f0bfcff26dbc141fe8a346ab6a1a3bc924794bfe3c9af306" -} diff --git a/backend/.sqlx/query-e38a66d15382703b99054a9b609af9f3854bb65393d33b265476c0e68aec4f61.json b/backend/.sqlx/query-e38a66d15382703b99054a9b609af9f3854bb65393d33b265476c0e68aec4f61.json deleted file mode 100644 index 78e5750992..0000000000 --- a/backend/.sqlx/query-e38a66d15382703b99054a9b609af9f3854bb65393d33b265476c0e68aec4f61.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE nats_trigger SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "e38a66d15382703b99054a9b609af9f3854bb65393d33b265476c0e68aec4f61" -} diff --git a/backend/.sqlx/query-f37140fcdc721a8b199471b30c2baf124affa2eaf56c801c8dac3264c584f981.json b/backend/.sqlx/query-e572fa64eec9188368d7c271ac7ecd6b45dc161423abab22cddb2b13f6fb9833.json similarity index 74% rename from backend/.sqlx/query-f37140fcdc721a8b199471b30c2baf124affa2eaf56c801c8dac3264c584f981.json rename to backend/.sqlx/query-e572fa64eec9188368d7c271ac7ecd6b45dc161423abab22cddb2b13f6fb9833.json index ace3edec94..2ee8c422fd 100644 --- a/backend/.sqlx/query-f37140fcdc721a8b199471b30c2baf124affa2eaf56c801c8dac3264c584f981.json +++ b/backend/.sqlx/query-e572fa64eec9188368d7c271ac7ecd6b45dc161423abab22cddb2b13f6fb9833.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, path, app_version) WHERE app_version IS NOT NULL DO UPDATE SET callback_job_ids = $4, deployment_msg = $5", + "query": "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, path, app_version) WHERE app_version IS NOT NULL DO UPDATE SET callback_job_ids = EXCLUDED.callback_job_ids, deployment_msg = EXCLUDED.deployment_msg", "describe": { "columns": [], "parameters": { @@ -14,5 +14,5 @@ }, "nullable": [] }, - "hash": "f37140fcdc721a8b199471b30c2baf124affa2eaf56c801c8dac3264c584f981" + "hash": "e572fa64eec9188368d7c271ac7ecd6b45dc161423abab22cddb2b13f6fb9833" } diff --git a/backend/.sqlx/query-6afc5c7cbb3abe11ade0cedf1f7328005ce4de3165cdd998e5a0d27e044c7153.json b/backend/.sqlx/query-e5f8830450e90f678494ae2b0f86b0692a02dec4c8415fd7d364b0cc088905c2.json similarity index 70% rename from backend/.sqlx/query-6afc5c7cbb3abe11ade0cedf1f7328005ce4de3165cdd998e5a0d27e044c7153.json rename to backend/.sqlx/query-e5f8830450e90f678494ae2b0f86b0692a02dec4c8415fd7d364b0cc088905c2.json index 0bbe880941..3867319977 100644 --- a/backend/.sqlx/query-6afc5c7cbb3abe11ade0cedf1f7328005ce4de3165cdd998e5a0d27e044c7153.json +++ b/backend/.sqlx/query-e5f8830450e90f678494ae2b0f86b0692a02dec4c8415fd7d364b0cc088905c2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5", + "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (worker) \n DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group", "describe": { "columns": [], "parameters": { @@ -18,5 +18,5 @@ }, "nullable": [] }, - "hash": "6afc5c7cbb3abe11ade0cedf1f7328005ce4de3165cdd998e5a0d27e044c7153" + "hash": "e5f8830450e90f678494ae2b0f86b0692a02dec4c8415fd7d364b0cc088905c2" } diff --git a/backend/.sqlx/query-e628e560caad2e6f37459a255cc931837dd6fb319577ba9d6103fb88b807bfca.json b/backend/.sqlx/query-e628e560caad2e6f37459a255cc931837dd6fb319577ba9d6103fb88b807bfca.json deleted file mode 100644 index b75aaa7383..0000000000 --- a/backend/.sqlx/query-e628e560caad2e6f37459a255cc931837dd6fb319577ba9d6103fb88b807bfca.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workspace_runnable_dependencies (\n flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id, app_path\n ) VALUES ($1, $2, $3, $4, $5, $6)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Int8", - "Bool", - "Varchar", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "e628e560caad2e6f37459a255cc931837dd6fb319577ba9d6103fb88b807bfca" -} diff --git a/backend/.sqlx/query-e6c4454e552dc82db1af2d6b887ebb4b78eb58600cc21e3ffb12dd9e5a0a6f08.json b/backend/.sqlx/query-e6c4454e552dc82db1af2d6b887ebb4b78eb58600cc21e3ffb12dd9e5a0a6f08.json new file mode 100644 index 0000000000..441b111a38 --- /dev/null +++ b/backend/.sqlx/query-e6c4454e552dc82db1af2d6b887ebb4b78eb58600cc21e3ffb12dd9e5a0a6f08.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM flow_conversation WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e6c4454e552dc82db1af2d6b887ebb4b78eb58600cc21e3ffb12dd9e5a0a6f08" +} diff --git a/backend/.sqlx/query-e734447a2506c7d69e744f8ddf1bbc3fad75c097ed483adba1f2f7f593481807.json b/backend/.sqlx/query-e734447a2506c7d69e744f8ddf1bbc3fad75c097ed483adba1f2f7f593481807.json deleted file mode 100644 index 3d0c982141..0000000000 --- a/backend/.sqlx/query-e734447a2506c7d69e744f8ddf1bbc3fad75c097ed483adba1f2f7f593481807.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO script (\n workspace_id, hash, path, parent_hashes, summary, description, content,\n created_by, created_at, archived, schema, deleted, is_template,\n extra_perms, lock, lock_error_logs, language, kind, tag, draft_only,\n envs, concurrent_limit, concurrency_time_window_s, cache_ttl,\n dedicated_worker, ws_error_handler_muted, priority, timeout,\n delete_after_use, restart_unless_cancelled, concurrency_key,\n visible_to_runner_only, no_main_func, codebase, has_preprocessor,\n on_behalf_of_email, assets\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13,\n $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24,\n $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35,\n $36, $37\n )", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Int8", - "Varchar", - "Int8Array", - "Text", - "Text", - "Text", - "Varchar", - "Timestamptz", - "Bool", - "Json", - "Bool", - "Bool", - "Jsonb", - "Text", - "Text", - { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java", - "duckdb", - "ruby" - ] - } - } - }, - { - "Custom": { - "name": "script_kind", - "kind": { - "Enum": [ - "script", - "trigger", - "failure", - "command", - "approval", - "preprocessor" - ] - } - } - }, - "Varchar", - "Bool", - "VarcharArray", - "Int4", - "Int4", - "Int4", - "Bool", - "Bool", - "Int2", - "Int4", - "Bool", - "Bool", - "Varchar", - "Bool", - "Bool", - "Varchar", - "Bool", - "Text", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "e734447a2506c7d69e744f8ddf1bbc3fad75c097ed483adba1f2f7f593481807" -} diff --git a/backend/.sqlx/query-e619fa013528a6275f98e14ae1727c55b0d4f4a5e4ee87c29251042e2916f0a0.json b/backend/.sqlx/query-e7c61bbdcf882f6e1e9b11df03e7a2ee318c72b365c0fe0b9fbec886a461f5e4.json similarity index 67% rename from backend/.sqlx/query-e619fa013528a6275f98e14ae1727c55b0d4f4a5e4ee87c29251042e2916f0a0.json rename to backend/.sqlx/query-e7c61bbdcf882f6e1e9b11df03e7a2ee318c72b365c0fe0b9fbec886a461f5e4.json index 2874c70c1c..6d41284328 100644 --- a/backend/.sqlx/query-e619fa013528a6275f98e14ae1727c55b0d4f4a5e4ee87c29251042e2916f0a0.json +++ b/backend/.sqlx/query-e7c61bbdcf882f6e1e9b11df03e7a2ee318c72b365c0fe0b9fbec886a461f5e4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO gcp_trigger (\n gcp_resource_path,\n subscription_id,\n topic_id,\n delivery_type,\n delivery_config,\n workspace_id, \n path, \n script_path, \n is_flow, \n email, \n enabled, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry,\n auto_acknowledge_msg\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4,\n $5,\n $6, \n $7, \n $8, \n $9,\n $10,\n $11,\n $12,\n $13,\n $14,\n $15,\n $16\n )", + "query": "\n INSERT INTO gcp_trigger (\n gcp_resource_path,\n subscription_id,\n topic_id,\n delivery_type,\n delivery_config,\n workspace_id, \n path, \n script_path, \n is_flow, \n email, \n enabled, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry,\n auto_acknowledge_msg,\n ack_deadline\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4,\n $5,\n $6, \n $7, \n $8, \n $9,\n $10,\n $11,\n $12,\n $13,\n $14,\n $15,\n $16,\n $17\n )", "describe": { "columns": [], "parameters": { @@ -30,10 +30,11 @@ "Varchar", "Jsonb", "Jsonb", - "Bool" + "Bool", + "Int4" ] }, "nullable": [] }, - "hash": "e619fa013528a6275f98e14ae1727c55b0d4f4a5e4ee87c29251042e2916f0a0" + "hash": "e7c61bbdcf882f6e1e9b11df03e7a2ee318c72b365c0fe0b9fbec886a461f5e4" } diff --git a/backend/.sqlx/query-e8925d199bd923dd7077b40477c5fcf2253407713692a88e6e7e51dfb59bed4d.json b/backend/.sqlx/query-e8925d199bd923dd7077b40477c5fcf2253407713692a88e6e7e51dfb59bed4d.json new file mode 100644 index 0000000000..5e83c3e715 --- /dev/null +++ b/backend/.sqlx/query-e8925d199bd923dd7077b40477c5fcf2253407713692a88e6e7e51dfb59bed4d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE \n flow \n SET \n ws_error_handler_muted = $3 \n WHERE \n path = $1 AND \n workspace_id = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "e8925d199bd923dd7077b40477c5fcf2253407713692a88e6e7e51dfb59bed4d" +} diff --git a/backend/.sqlx/query-2367e7c0f7fbafe0971a187c0909617da55251e97180babf6ac9e8068f26d73d.json b/backend/.sqlx/query-e8e33f599eae064011232f9f715e676d0c8ae31982865cb2c8103ed735c42c69.json similarity index 74% rename from backend/.sqlx/query-2367e7c0f7fbafe0971a187c0909617da55251e97180babf6ac9e8068f26d73d.json rename to backend/.sqlx/query-e8e33f599eae064011232f9f715e676d0c8ae31982865cb2c8103ed735c42c69.json index d643e4b8f8..84645b71b0 100644 --- a/backend/.sqlx/query-2367e7c0f7fbafe0971a187c0909617da55251e97180babf6ac9e8068f26d73d.json +++ b/backend/.sqlx/query-e8e33f599eae064011232f9f715e676d0c8ae31982865cb2c8103ed735c42c69.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO deployment_metadata (workspace_id, path, flow_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL DO UPDATE SET callback_job_ids = $4, deployment_msg = $5", + "query": "INSERT INTO deployment_metadata (workspace_id, path, flow_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL DO UPDATE SET callback_job_ids = EXCLUDED.callback_job_ids, deployment_msg = EXCLUDED.deployment_msg", "describe": { "columns": [], "parameters": { @@ -14,5 +14,5 @@ }, "nullable": [] }, - "hash": "2367e7c0f7fbafe0971a187c0909617da55251e97180babf6ac9e8068f26d73d" + "hash": "e8e33f599eae064011232f9f715e676d0c8ae31982865cb2c8103ed735c42c69" } diff --git a/backend/.sqlx/query-ea9385509319f66b9330221eb50b85edafb9408d0306f17bb78b65a5d81c570b.json b/backend/.sqlx/query-ea9385509319f66b9330221eb50b85edafb9408d0306f17bb78b65a5d81c570b.json new file mode 100644 index 0000000000..ab648f579d --- /dev/null +++ b/backend/.sqlx/query-ea9385509319f66b9330221eb50b85edafb9408d0306f17bb78b65a5d81c570b.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DROP INDEX CONCURRENTLY IF EXISTS ix_v2_job_root_by_path", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "ea9385509319f66b9330221eb50b85edafb9408d0306f17bb78b65a5d81c570b" +} diff --git a/backend/.sqlx/query-ead84a63cb965e36155605434c9e3670e50344f39b0d61eef8e60c2cdb57ae1f.json b/backend/.sqlx/query-ead84a63cb965e36155605434c9e3670e50344f39b0d61eef8e60c2cdb57ae1f.json new file mode 100644 index 0000000000..499e2e6730 --- /dev/null +++ b/backend/.sqlx/query-ead84a63cb965e36155605434c9e3670e50344f39b0d61eef8e60c2cdb57ae1f.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM dependency_map\n WHERE workspace_id = $1\n AND importer_path = $2\n AND importer_kind = $3::text::IMPORTER_KIND\n AND importer_node_id = $4\n AND imported_path = $5\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ead84a63cb965e36155605434c9e3670e50344f39b0d61eef8e60c2cdb57ae1f" +} diff --git a/backend/.sqlx/query-eb110e722ba8ac32d9d69010dc7e5f5763c55a5ac255f939f1fda8d3f9200f8d.json b/backend/.sqlx/query-eb110e722ba8ac32d9d69010dc7e5f5763c55a5ac255f939f1fda8d3f9200f8d.json new file mode 100644 index 0000000000..6b2a36ec10 --- /dev/null +++ b/backend/.sqlx/query-eb110e722ba8ac32d9d69010dc7e5f5763c55a5ac255f939f1fda8d3f9200f8d.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, is_admin, operator, added_via FROM usr WHERE username = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_admin", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "operator", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "added_via", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "eb110e722ba8ac32d9d69010dc7e5f5763c55a5ac255f939f1fda8d3f9200f8d" +} diff --git a/backend/.sqlx/query-ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b.json b/backend/.sqlx/query-ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b.json deleted file mode 100644 index d87e680abe..0000000000 --- a/backend/.sqlx/query-ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS (\n SELECT 1 FROM v2_as_completed_job\n WHERE workspace_id = $2\n AND (job_kind = 'appscript' OR job_kind = 'preview')\n AND created_by = 'anonymous'\n AND started_at > now() - interval '3 hours'\n AND script_path LIKE $3 || '/%'\n AND result @> ('{\"s3\":\"' || $1 || '\"}')::jsonb\n )", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b" -} diff --git a/backend/.sqlx/query-edd57b3d59ddc21b99212b5fabd4a8b793c4ae618a04220b3609c8c3c168f8fd.json b/backend/.sqlx/query-edd57b3d59ddc21b99212b5fabd4a8b793c4ae618a04220b3609c8c3c168f8fd.json new file mode 100644 index 0000000000..f67e52ab6b --- /dev/null +++ b/backend/.sqlx/query-edd57b3d59ddc21b99212b5fabd4a8b793c4ae618a04220b3609c8c3c168f8fd.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "executions", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "edd57b3d59ddc21b99212b5fabd4a8b793c4ae618a04220b3609c8c3c168f8fd" +} diff --git a/backend/.sqlx/query-84d048ea323758842dc564c700b524d1a5b196a7b77afcb02f46b84b22088bbf.json b/backend/.sqlx/query-f03d52c091d10d27a274cebaf370a4665a518da0560fd9ed7add05c66da3898f.json similarity index 51% rename from backend/.sqlx/query-84d048ea323758842dc564c700b524d1a5b196a7b77afcb02f46b84b22088bbf.json rename to backend/.sqlx/query-f03d52c091d10d27a274cebaf370a4665a518da0560fd9ed7add05c66da3898f.json index e4473b360c..c5297f1499 100644 --- a/backend/.sqlx/query-84d048ea323758842dc564c700b524d1a5b196a7b77afcb02f46b84b22088bbf.json +++ b/backend/.sqlx/query-f03d52c091d10d27a274cebaf370a4665a518da0560fd9ed7add05c66da3898f.json @@ -1,20 +1,20 @@ { "db_name": "PostgreSQL", - "query": "SELECT value FROM global_settings WHERE name = 'teams'", + "query": "SELECT value::text FROM app_version WHERE id = 0 AND app_id = 2", "describe": { "columns": [ { "ordinal": 0, "name": "value", - "type_info": "Jsonb" + "type_info": "Text" } ], "parameters": { "Left": [] }, "nullable": [ - false + null ] }, - "hash": "84d048ea323758842dc564c700b524d1a5b196a7b77afcb02f46b84b22088bbf" + "hash": "f03d52c091d10d27a274cebaf370a4665a518da0560fd9ed7add05c66da3898f" } diff --git a/backend/.sqlx/query-f050be8da0d99aa1af64f5d32a8df01c5a59b036b2669878cc557537efb492c8.json b/backend/.sqlx/query-f050be8da0d99aa1af64f5d32a8df01c5a59b036b2669878cc557537efb492c8.json deleted file mode 100644 index 7334c70fb4..0000000000 --- a/backend/.sqlx/query-f050be8da0d99aa1af64f5d32a8df01c5a59b036b2669878cc557537efb492c8.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'mqtt'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "f050be8da0d99aa1af64f5d32a8df01c5a59b036b2669878cc557537efb492c8" -} diff --git a/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json b/backend/.sqlx/query-f06ab5e0369b35694fa02c3aac685bd547a1d271eb7401df57fe1774de3211bf.json similarity index 82% rename from backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json rename to backend/.sqlx/query-f06ab5e0369b35694fa02c3aac685bd547a1d271eb7401df57fe1774de3211bf.json index 60d2cfdc01..4e7b02738e 100644 --- a/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json +++ b/backend/.sqlx/query-f06ab5e0369b35694fa02c3aac685bd547a1d271eb7401df57fe1774de3211bf.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by, path from script where hash = $1 AND workspace_id = $2", + "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, debounce_key, debounce_delay_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by, path from script where hash = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -30,11 +30,21 @@ }, { "ordinal": 5, + "name": "debounce_key", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "debounce_delay_s", + "type_info": "Int4" + }, + { + "ordinal": 7, "name": "cache_ttl", "type_info": "Int4" }, { - "ordinal": 6, + "ordinal": 8, "name": "language: ScriptLang", "type_info": { "Custom": { @@ -70,42 +80,42 @@ } }, { - "ordinal": 7, + "ordinal": 9, "name": "dedicated_worker", "type_info": "Bool" }, { - "ordinal": 8, + "ordinal": 10, "name": "priority", "type_info": "Int2" }, { - "ordinal": 9, + "ordinal": 11, "name": "delete_after_use", "type_info": "Bool" }, { - "ordinal": 10, + "ordinal": 12, "name": "timeout", "type_info": "Int4" }, { - "ordinal": 11, + "ordinal": 13, "name": "has_preprocessor", "type_info": "Bool" }, { - "ordinal": 12, + "ordinal": 14, "name": "on_behalf_of_email", "type_info": "Text" }, { - "ordinal": 13, + "ordinal": 15, "name": "created_by", "type_info": "Varchar" }, { - "ordinal": 14, + "ordinal": 16, "name": "path", "type_info": "Varchar" } @@ -123,6 +133,8 @@ true, true, true, + true, + true, false, true, true, @@ -134,5 +146,5 @@ false ] }, - "hash": "0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936" + "hash": "f06ab5e0369b35694fa02c3aac685bd547a1d271eb7401df57fe1774de3211bf" } diff --git a/backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json b/backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json deleted file mode 100644 index a1ce88bd88..0000000000 --- a/backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE last_client_ping > NOW() - INTERVAL '10 seconds' AND workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'nats' AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2" -} diff --git a/backend/.sqlx/query-f17c9fed09897191ee70214223d4b83a05f1d649bfedff9f24cc3e6c702d42df.json b/backend/.sqlx/query-f17c9fed09897191ee70214223d4b83a05f1d649bfedff9f24cc3e6c702d42df.json new file mode 100644 index 0000000000..a68b5dd969 --- /dev/null +++ b/backend/.sqlx/query-f17c9fed09897191ee70214223d4b83a05f1d649bfedff9f24cc3e6c702d42df.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_job_v2_job_root_by_path_2 ON v2_job (workspace_id, runnable_path) WHERE parent_job IS NULL;", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "f17c9fed09897191ee70214223d4b83a05f1d649bfedff9f24cc3e6c702d42df" +} diff --git a/backend/.sqlx/query-f1fe8508f6cd29d4c0d354473529fc46f506ddf98d45fc870e2855c4a4b424ea.json b/backend/.sqlx/query-f1fe8508f6cd29d4c0d354473529fc46f506ddf98d45fc870e2855c4a4b424ea.json new file mode 100644 index 0000000000..effa800640 --- /dev/null +++ b/backend/.sqlx/query-f1fe8508f6cd29d4c0d354473529fc46f506ddf98d45fc870e2855c4a4b424ea.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO debounce_key (key, job_id)\n VALUES ($1, $2)\n ON CONFLICT (key)\n DO UPDATE SET job_id = debounce_key.job_id -- No actual change, just to trigger UPDATE\n RETURNING CASE WHEN xmax != 0 THEN job_id ELSE NULL END AS job_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f1fe8508f6cd29d4c0d354473529fc46f506ddf98d45fc870e2855c4a4b424ea" +} diff --git a/backend/.sqlx/query-f5568a691ec5931634cf986f806f5eae7bb8ed0f5c6e54ca3f49a991c53ed50d.json b/backend/.sqlx/query-f5568a691ec5931634cf986f806f5eae7bb8ed0f5c6e54ca3f49a991c53ed50d.json new file mode 100644 index 0000000000..68461c0c98 --- /dev/null +++ b/backend/.sqlx/query-f5568a691ec5931634cf986f806f5eae7bb8ed0f5c6e54ca3f49a991c53ed50d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status \n SET flow_status = jsonb_set(\n flow_status,\n '{memory_id}',\n to_jsonb($2::uuid)\n )\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "f5568a691ec5931634cf986f806f5eae7bb8ed0f5c6e54ca3f49a991c53ed50d" +} diff --git a/backend/.sqlx/query-f830e934b43a83f857f682383f36ed56df6518e3035778e4e3b95182d2504e00.json b/backend/.sqlx/query-f830e934b43a83f857f682383f36ed56df6518e3035778e4e3b95182d2504e00.json new file mode 100644 index 0000000000..8025c7654c --- /dev/null +++ b/backend/.sqlx/query-f830e934b43a83f857f682383f36ed56df6518e3035778e4e3b95182d2504e00.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET slack_oauth_client_id = $1, slack_oauth_client_secret = $2\n WHERE workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "f830e934b43a83f857f682383f36ed56df6518e3035778e4e3b95182d2504e00" +} diff --git a/backend/.sqlx/query-f8ac5379ecfbff7b8ae75c821680737b249a64c8d9e8f7dbcc46fce98e874571.json b/backend/.sqlx/query-f8ac5379ecfbff7b8ae75c821680737b249a64c8d9e8f7dbcc46fce98e874571.json new file mode 100644 index 0000000000..70fbb752eb --- /dev/null +++ b/backend/.sqlx/query-f8ac5379ecfbff7b8ae75c821680737b249a64c8d9e8f7dbcc46fce98e874571.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, concurrency_key, log_file, metrics", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "f8ac5379ecfbff7b8ae75c821680737b249a64c8d9e8f7dbcc46fce98e874571" +} diff --git a/backend/.sqlx/query-f9e1334ecb17b313924587d96fc8b0310b14d75e85a558c8cb70f2761923d175.json b/backend/.sqlx/query-f9e1334ecb17b313924587d96fc8b0310b14d75e85a558c8cb70f2761923d175.json deleted file mode 100644 index f21a486c83..0000000000 --- a/backend/.sqlx/query-f9e1334ecb17b313924587d96fc8b0310b14d75e85a558c8cb70f2761923d175.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "f9e1334ecb17b313924587d96fc8b0310b14d75e85a558c8cb70f2761923d175" -} diff --git a/backend/.sqlx/query-526bfaccaafbe2e6f70dd5e6cd21c0c60d4ec155f79d067a8b74cf24eebad88c.json b/backend/.sqlx/query-fad966db585b91c9ce143c9aa26a826aec1ddb193a7f4988c5f12b1a2d8ce071.json similarity index 58% rename from backend/.sqlx/query-526bfaccaafbe2e6f70dd5e6cd21c0c60d4ec155f79d067a8b74cf24eebad88c.json rename to backend/.sqlx/query-fad966db585b91c9ce143c9aa26a826aec1ddb193a7f4988c5f12b1a2d8ce071.json index 08e38953ab..673a98eade 100644 --- a/backend/.sqlx/query-526bfaccaafbe2e6f70dd5e6cd21c0c60d4ec155f79d067a8b74cf24eebad88c.json +++ b/backend/.sqlx/query-fad966db585b91c9ce143c9aa26a826aec1ddb193a7f4988c5f12b1a2d8ce071.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT versions[array_upper(versions, 1)] FROM flow WHERE path = $1 AND workspace_id = $2", + "query": "SELECT app.versions[array_upper(app.versions, 1)] FROM app\n WHERE app.path = $1 AND app.workspace_id = $2", "describe": { "columns": [ { @@ -19,5 +19,5 @@ null ] }, - "hash": "526bfaccaafbe2e6f70dd5e6cd21c0c60d4ec155f79d067a8b74cf24eebad88c" + "hash": "fad966db585b91c9ce143c9aa26a826aec1ddb193a7f4988c5f12b1a2d8ce071" } diff --git a/backend/.sqlx/query-3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e.json b/backend/.sqlx/query-fbd38c9c4f4ecba2d3e3d79433327fd4280466eec443aff4f1e8c1ab810cca7e.json similarity index 51% rename from backend/.sqlx/query-3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e.json rename to backend/.sqlx/query-fbd38c9c4f4ecba2d3e3d79433327fd4280466eec443aff4f1e8c1ab810cca7e.json index 5c5d9b68ba..29cedfbc1c 100644 --- a/backend/.sqlx/query-3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e.json +++ b/backend/.sqlx/query-fbd38c9c4f4ecba2d3e3d79433327fd4280466eec443aff4f1e8c1ab810cca7e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH job_info AS (\n -- Query for Teams (running jobs)\n SELECT\n parent.job_kind AS \"job_kind!: JobKind\",\n parent.script_hash AS \"script_hash: ScriptHash\",\n parent.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n child.parent_job AS \"parent_job: Uuid\",\n parent.created_at AS \"created_at!: chrono::NaiveDateTime\",\n parent.created_by AS \"created_by!\",\n parent.script_path,\n parent.args AS \"args: sqlx::types::Json>\"\n FROM v2_as_queue child\n JOIN v2_as_queue parent ON parent.id = child.parent_job\n WHERE child.id = $1 AND child.workspace_id = $2\n UNION ALL\n -- Query for Slack (completed jobs)\n SELECT\n v2_as_queue.job_kind AS \"job_kind!: JobKind\",\n v2_as_queue.script_hash AS \"script_hash: ScriptHash\",\n v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n v2_as_completed_job.parent_job AS \"parent_job: Uuid\",\n v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\",\n v2_as_completed_job.created_by AS \"created_by!\",\n v2_as_queue.script_path,\n v2_as_queue.args AS \"args: sqlx::types::Json>\"\n FROM v2_as_queue\n JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id\n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2\n )\n SELECT * FROM job_info LIMIT 1", + "query": "WITH job_info AS (\n -- Query for Teams (running jobs)\n SELECT\n parent_j.kind AS \"job_kind!: JobKind\",\n parent_j.runnable_id AS \"script_hash: ScriptHash\",\n parent_j.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n child_j.parent_job AS \"parent_job: Uuid\",\n parent_j.created_at AS \"created_at!: chrono::NaiveDateTime\",\n parent_j.created_by AS \"created_by!\",\n parent_j.runnable_path as script_path,\n parent_j.args AS \"args: sqlx::types::Json>\"\n FROM v2_job_queue child_q JOIN v2_job child_j USING (id)\n JOIN v2_job parent_j ON parent_j.id = child_j.parent_job\n WHERE child_j.id = $1 AND child_j.workspace_id = $2\n UNION ALL\n -- Query for Slack (completed jobs)\n SELECT\n parent_j.kind AS \"job_kind!: JobKind\",\n parent_j.runnable_id AS \"script_hash: ScriptHash\",\n parent_j.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n completed_j.parent_job AS \"parent_job: Uuid\",\n completed_j.created_at AS \"created_at!: chrono::NaiveDateTime\",\n completed_j.created_by AS \"created_by!\",\n parent_j.runnable_path as script_path,\n parent_j.args AS \"args: sqlx::types::Json>\"\n FROM v2_job_completed completed_c JOIN v2_job completed_j USING (id)\n JOIN v2_job parent_j ON parent_j.id = completed_j.parent_job\n WHERE completed_j.id = $1 AND completed_j.workspace_id = $2\n )\n SELECT * FROM job_info LIMIT 1", "describe": { "columns": [ { @@ -25,7 +25,7 @@ "noop", "appdependencies", "deploymentcallback", - "singlescriptflow", + "singlestepflow", "flowscript", "flownode", "appscript", @@ -88,5 +88,5 @@ null ] }, - "hash": "3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e" + "hash": "fbd38c9c4f4ecba2d3e3d79433327fd4280466eec443aff4f1e8c1ab810cca7e" } diff --git a/backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json b/backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json deleted file mode 100644 index 091d205b40..0000000000 --- a/backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'mqtt' AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96" -} diff --git a/backend/.sqlx/query-fd0a0ca4a107dc813240ad71f372ab7e0bf26431158fbfb5f0023ed847ca7dc5.json b/backend/.sqlx/query-fd0a0ca4a107dc813240ad71f372ab7e0bf26431158fbfb5f0023ed847ca7dc5.json deleted file mode 100644 index d47bb3fd68..0000000000 --- a/backend/.sqlx/query-fd0a0ca4a107dc813240ad71f372ab7e0bf26431158fbfb5f0023ed847ca7dc5.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n gcp_trigger \n SET \n enabled = FALSE, \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "fd0a0ca4a107dc813240ad71f372ab7e0bf26431158fbfb5f0023ed847ca7dc5" -} diff --git a/backend/.sqlx/query-fd403acc343182fdab100263f8ef8067c1b43a97ea96845d92e2cbf85fbd6311.json b/backend/.sqlx/query-fd403acc343182fdab100263f8ef8067c1b43a97ea96845d92e2cbf85fbd6311.json new file mode 100644 index 0000000000..fbd0a96f98 --- /dev/null +++ b/backend/.sqlx/query-fd403acc343182fdab100263f8ef8067c1b43a97ea96845d92e2cbf85fbd6311.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, versions[array_upper(versions, 1)] as version FROM app WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "version", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "fd403acc343182fdab100263f8ef8067c1b43a97ea96845d92e2cbf85fbd6311" +} diff --git a/backend/.sqlx/query-fd55112d55995ab08d2c275aa6430cdec1cacebdf2f2b3dd6f678b434643eb50.json b/backend/.sqlx/query-fd55112d55995ab08d2c275aa6430cdec1cacebdf2f2b3dd6f678b434643eb50.json new file mode 100644 index 0000000000..b8eb1b000f --- /dev/null +++ b/backend/.sqlx/query-fd55112d55995ab08d2c275aa6430cdec1cacebdf2f2b3dd6f678b434643eb50.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE global_settings SET value = jsonb_set(value, '{instance_catalog_db_status}', (COALESCE(value->'instance_catalog_db_status', '{}'::jsonb) || to_jsonb($1::json))) WHERE name = 'ducklake_settings'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Json" + ] + }, + "nullable": [] + }, + "hash": "fd55112d55995ab08d2c275aa6430cdec1cacebdf2f2b3dd6f678b434643eb50" +} diff --git a/backend/.sqlx/query-febd70e6d5304ad912c232d3d8fc3f8ce0133d8a1a9e5ca1124eff9a988dc2d7.json b/backend/.sqlx/query-febd70e6d5304ad912c232d3d8fc3f8ce0133d8a1a9e5ca1124eff9a988dc2d7.json deleted file mode 100644 index 7958f9d2f3..0000000000 --- a/backend/.sqlx/query-febd70e6d5304ad912c232d3d8fc3f8ce0133d8a1a9e5ca1124eff9a988dc2d7.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE websocket_trigger SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND server_id = $4 AND enabled IS TRUE RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "febd70e6d5304ad912c232d3d8fc3f8ce0133d8a1a9e5ca1124eff9a988dc2d7" -} diff --git a/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json b/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json deleted file mode 100644 index 0e42bd0fdb..0000000000 --- a/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job (id, runnable_id, runnable_path, kind, script_lang, tag, created_by, permissioned_as, permissioned_as_email, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9 FROM generate_series(1, $10)) RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Int8", - "Varchar", - { - "Custom": { - "name": "job_kind", - "kind": { - "Enum": [ - "script", - "preview", - "flow", - "dependencies", - "flowpreview", - "script_hub", - "identity", - "flowdependencies", - "http", - "graphql", - "postgresql", - "noop", - "appdependencies", - "deploymentcallback", - "singlescriptflow", - "flowscript", - "flownode", - "appscript", - "aiagent" - ] - } - } - }, - { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java", - "duckdb", - "ruby" - ] - } - } - }, - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Int4" - ] - }, - "nullable": [ - false - ] - }, - "hash": "ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238" -} diff --git a/backend/.sqlx/query-ffb6b2a40b605f9c55826a0e519af80b100d2ffa58ec567223e27099d364fcc8.json b/backend/.sqlx/query-ffb6b2a40b605f9c55826a0e519af80b100d2ffa58ec567223e27099d364fcc8.json deleted file mode 100644 index c75cb3385a..0000000000 --- a/backend/.sqlx/query-ffb6b2a40b605f9c55826a0e519af80b100d2ffa58ec567223e27099d364fcc8.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'nats'", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "ffb6b2a40b605f9c55826a0e519af80b100d2ffa58ec567223e27099d364fcc8" -} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index efd189e08d..d3a95f582b 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14,9 +14,9 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" dependencies = [ "gimli", ] @@ -122,7 +122,7 @@ checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", "const-random", - "getrandom 0.3.3", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -130,9 +130,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -169,9 +169,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.20" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" dependencies = [ "anstyle", "anstyle-parse", @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.11" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" @@ -219,9 +219,18 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.99" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "ar_archive_writer" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c269894b6fe5e9d7ada0cf69b5bf847ff35bc25fc271f08e1d080fce80339a" +dependencies = [ + "object 0.32.2", +] [[package]] name = "arbitrary" @@ -487,7 +496,7 @@ dependencies = [ "memchr", "num", "regex", - "regex-syntax 0.8.6", + "regex-syntax 0.8.8", ] [[package]] @@ -547,7 +556,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -620,7 +629,7 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.4", "tokio-util", "tokio-websockets", "tracing", @@ -638,7 +647,7 @@ dependencies = [ "bytes", "http 1.3.1", "rand 0.8.5", - "reqwest 0.12.23", + "reqwest 0.12.24", "serde", "serde-aux", "serde_json", @@ -661,7 +670,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -683,7 +692,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -694,7 +703,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -779,9 +788,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.6" +version = "1.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d025db5d9f52cbc413b167136afb3d8aeea708c0d8884783cf6253be5e22f6f2" +checksum = "faf26925f4a5b59eb76722b63c2892b1d70d06fa053c72e4a100ec308c1d47bc" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -791,9 +800,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b8ff6c09cd57b16da53641caa860168b88c172a5ee163b0288d3d6eea12786" +checksum = "879b6c89592deb404ba4dc0ae6b58ffd1795c78991cbb5b8bc441c48a070440d" dependencies = [ "aws-lc-sys", "zeroize", @@ -801,9 +810,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.31.0" +version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e44d16778acaf6a9ec9899b92cebd65580b83f685446bf2e1f5d3d732f99dcd" +checksum = "107a4e9d9cab9963e04e84bb8dee0e25f2a987f9a8bad5ed054abd439caa8f8c" dependencies = [ "bindgen 0.72.1", "cc", @@ -950,9 +959,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.3.4" +version = "1.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084c34162187d39e3740cb635acd73c4e3a551a36146ad6fe8883c929c9f876c" +checksum = "c35452ec3f001e1f2f6db107b6373f1f48f05ec63ba2c5c9fa91f07dad32af11" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -972,9 +981,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.5" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" +checksum = "127fcfad33b7dfc531141fda7e1c402ac65f88aca5511a4d31e2e3d2cd01ce9c" dependencies = [ "futures-util", "pin-project-lite", @@ -983,15 +992,16 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.62.3" +version = "0.62.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c4dacf2d38996cf729f55e7a762b30918229917eca115de45dfa8dfb97796c9" +checksum = "445d5d720c99eed0b4aa674ed00d835d9b1427dd73e04adaf2f94c6b2d6f9fca" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "bytes-utils", "futures-core", + "futures-util", "http 0.2.12", "http 1.3.1", "http-body 0.4.6", @@ -1023,7 +1033,7 @@ dependencies = [ "pin-project-lite", "rustls 0.21.12", "rustls 0.23.29", - "rustls-native-certs 0.8.1", + "rustls-native-certs 0.8.2", "rustls-pki-types", "tokio", "tower 0.5.2", @@ -1032,27 +1042,27 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.61.5" +version = "0.61.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaa31b350998e703e9826b2104dd6f63be0508666e1aba88137af060e8944047" +checksum = "2db31f727935fc63c6eeae8b37b438847639ec330a9161ece694efba257e0c54" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-observability" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9364d5989ac4dd918e5cc4c4bdcc61c9be17dcd2586ea7f69e348fc7c6cab393" +checksum = "2d1881b1ea6d313f9890710d65c158bdab6fb08c91ea825f74c1c8c357baf4cc" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.60.7" +version = "0.60.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +checksum = "d28a63441360c477465f80c7abac3b9c4d075ca638f982e605b7dc2a2c7156c9" dependencies = [ "aws-smithy-types", "urlencoding", @@ -1084,9 +1094,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.9.0" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07f5e0fc8a6b3f2303f331b94504bbf754d85488f402d6f1dd7a6080f99afe56" +checksum = "ec7204f9fd94749a7c53b26da1b961b4ac36bf070ef1e0b94bb09f79d4f6c193" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -1101,9 +1111,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.3.2" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8" +checksum = "25f535879a207fce0db74b679cfc3e91a3159c8144d717d55f5832aea9eef46e" dependencies = [ "base64-simd 0.8.0", "bytes", @@ -1127,9 +1137,9 @@ dependencies = [ [[package]] name = "aws-smithy-types-convert" -version = "0.60.9" +version = "0.60.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df786cc1aea35d24b609f7a32d05570916edfe7b3e09e81f2faf365f9062f647" +checksum = "c99945c7033e37bfe8dc76c19862ebd7544b14b4f34a8551ccf2690b433eba0e" dependencies = [ "aws-smithy-types", "chrono", @@ -1137,18 +1147,18 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.10" +version = "0.60.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db87b96cb1b16c024980f133968d52882ca0daaee3a086c6decc500f6c99728" +checksum = "eab77cdd036b11056d2a30a7af7b775789fb024bf216acc13884c6c97752ae56" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.8" +version = "1.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b069d19bf01e46298eaedd7c6f283fe565a59263e53eebec945f3e6398f42390" +checksum = "d79fb68e3d7fe5d4833ea34dc87d2e97d26d3086cb3da660bb6b1f76d98680b6" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1166,6 +1176,7 @@ checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", "axum-core", + "axum-macros", "bytes", "futures-util", "http 1.3.1", @@ -1214,6 +1225,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-macros" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.109", +] + [[package]] name = "az" version = "1.2.1" @@ -1222,9 +1244,9 @@ checksum = "7b7e4c2464d97fe331d41de9d5db0def0a96f4d823b8b32a2efd503578988973" [[package]] name = "backon" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ "fastrand", "gloo-timers", @@ -1233,17 +1255,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.75" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" dependencies = [ "addr2line", "cfg-if", "libc", "miniz_oxide 0.8.9", - "object", + "object 0.37.3", "rustc-demangle", - "windows-targets 0.52.6", + "windows-link 0.2.1", ] [[package]] @@ -1312,9 +1334,9 @@ dependencies = [ [[package]] name = "bigdecimal" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" +checksum = "560f42649de9fa436b73517378a147ec21f6c997a546581df4b4b31677828934" dependencies = [ "autocfg", "libm", @@ -1351,7 +1373,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.106", + "syn 2.0.109", "which 4.4.2", ] @@ -1372,7 +1394,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -1392,7 +1414,7 @@ dependencies = [ "regex", "rustc-hash 2.1.1", "shlex", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -1558,7 +1580,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-util", "tower-service", @@ -1579,9 +1601,9 @@ dependencies = [ [[package]] name = "bon" -version = "3.7.2" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2529c31017402be841eb45892278a6c21a000c0a17643af326c73a73f83f0fb" +checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1" dependencies = [ "bon-macros", "rustversion", @@ -1589,9 +1611,9 @@ dependencies = [ [[package]] name = "bon-macros" -version = "3.7.2" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82020dadcb845a345591863adb65d74fa8dc5c18a0b6d408470e13b7adc7005" +checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" dependencies = [ "darling 0.21.3", "ident_case", @@ -1599,7 +1621,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -1622,7 +1644,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -1632,7 +1654,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17d4f95e880cfd28c4ca5a006cf7f6af52b4bcb7b5866f573b2faa126fb7affb" dependencies = [ "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -1690,9 +1712,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" dependencies = [ "memchr", "serde", @@ -1751,22 +1773,22 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.23.2" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.10.1" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f154e572231cb6ba2bd1176980827e3d5dc04cc183a75dea38109fbdd672d29" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -1834,7 +1856,7 @@ dependencies = [ "byteorder", "gemm 0.17.1", "half", - "memmap2 0.9.8", + "memmap2 0.9.9", "num-traits", "num_cpus", "rand 0.9.0", @@ -1909,7 +1931,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b4a6cae9efc04cc6cbb8faf338d2c497c165c83e74509cf4dbedea948bbf6e5" dependencies = [ "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -1923,9 +1945,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.36" +version = "1.2.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5252b3d2648e5eedbc1a6f501e3c795e07025c1e93bbf8bbdd6eef7f447a6d54" +checksum = "37521ac7aabe3d13122dc382493e20c9416f299d2ccd5b3a5340a2570cdeb0f3" dependencies = [ "find-msvc-tools", "jobserver", @@ -1950,9 +1972,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" @@ -1978,7 +2000,7 @@ dependencies = [ "pure-rust-locales", "serde", "wasm-bindgen", - "windows-link 0.2.0", + "windows-link 0.2.1", ] [[package]] @@ -2027,14 +2049,14 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", - "libloading 0.8.8", + "libloading 0.8.9", ] [[package]] name = "clap" -version = "4.5.47" +version = "4.5.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" +checksum = "4c26d721170e0295f191a69bd9a1f93efcdb0aff38684b61ab5750468972e5f5" dependencies = [ "clap_builder", "clap_derive", @@ -2042,9 +2064,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.47" +version = "4.5.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" +checksum = "75835f0c7bf681bfd05abe44e965760fea999a5286c6eb2d59883634fd02011a" dependencies = [ "anstream", "anstyle", @@ -2054,21 +2076,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.47" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] name = "clap_lex" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "clipboard-win" @@ -2116,7 +2138,7 @@ dependencies = [ "nom 7.1.3", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -2138,7 +2160,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b03b7db8e0b4b2fdad6c551e634134e99ec000e5c8c3b6856c65e8bbaded7a3b" dependencies = [ "unicode-segmentation", - "unicode-width 0.2.1", + "unicode-width 0.2.2", ] [[package]] @@ -2165,7 +2187,7 @@ dependencies = [ "encode_unicode", "libc", "once_cell", - "unicode-width 0.2.1", + "unicode-width 0.2.2", "windows-sys 0.59.0", ] @@ -2197,9 +2219,9 @@ dependencies = [ [[package]] name = "const_format" -version = "0.2.34" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd" +checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad" dependencies = [ "const_format_proc_macros", "konst", @@ -2458,9 +2480,9 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" dependencies = [ "memchr", ] @@ -2498,7 +2520,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -2508,7 +2530,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b28bfe653d79bd16c77f659305b195b82bb5ce0c0eb2a4846b82ddbd77586813" dependencies = [ "bitflags 2.9.4", - "libloading 0.8.8", + "libloading 0.8.9", "winapi", ] @@ -2591,7 +2613,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -2605,7 +2627,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -2638,7 +2660,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -2649,7 +2671,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -2662,7 +2684,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core 0.9.11", + "parking_lot_core 0.9.12", ] [[package]] @@ -2676,7 +2698,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core 0.9.11", + "parking_lot_core 0.9.12", ] [[package]] @@ -2733,7 +2755,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "parquet", "rand 0.8.5", "regex", @@ -2768,7 +2790,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "tokio", ] @@ -2941,7 +2963,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "parquet", "rand 0.8.5", "tokio", @@ -2966,7 +2988,7 @@ dependencies = [ "futures", "log", "object_store", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "rand 0.8.5", "tempfile", "url", @@ -3102,7 +3124,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-plan", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "paste", ] @@ -3141,7 +3163,7 @@ checksum = "df6f88d7ee27daf8b108ba910f9015176b36fbc72902b1ca5c2a5f1d1717e1a1" dependencies = [ "datafusion-expr", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -3160,7 +3182,7 @@ dependencies = [ "log", "recursive", "regex", - "regex-syntax 0.8.6", + "regex-syntax 0.8.8", ] [[package]] @@ -3243,7 +3265,7 @@ dependencies = [ "indexmap 2.11.1", "itertools 0.14.0", "log", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "pin-project-lite", "tokio", ] @@ -3268,7 +3290,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "tokio", ] @@ -3342,7 +3364,7 @@ dependencies = [ "swc_visit", "swc_visit_macros", "text_lines", - "thiserror 2.0.16", + "thiserror 2.0.17", "unicode-width 0.1.14", "url", ] @@ -3356,7 +3378,7 @@ dependencies = [ "async-trait", "deno_core", "deno_error", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "uuid", ] @@ -3373,7 +3395,7 @@ dependencies = [ "rusqlite", "serde", "sha2 0.10.9", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", ] @@ -3397,7 +3419,7 @@ dependencies = [ "indexmap 2.11.1", "log", "once_cell", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "serde", "serde_json", "sha2 0.10.9", @@ -3417,7 +3439,7 @@ dependencies = [ "deno_webgpu", "image", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -3443,7 +3465,7 @@ dependencies = [ "serde", "serde_json", "sys_traits", - "thiserror 2.0.16", + "thiserror 2.0.17", "url", ] @@ -3479,7 +3501,7 @@ dependencies = [ "indexmap 2.11.1", "libc", "memoffset", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "percent-encoding", "pin-project", "serde", @@ -3488,7 +3510,7 @@ dependencies = [ "smallvec", "sourcemap 8.0.1", "static_assertions", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "url", "v8", @@ -3513,7 +3535,7 @@ dependencies = [ "deno_core", "deno_error", "saffron", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", ] @@ -3551,7 +3573,7 @@ dependencies = [ "sha2 0.10.9", "signature", "spki", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "uuid", "x25519-dalek", @@ -3579,7 +3601,7 @@ checksum = "babccedee31ce7e57c3e6dff2cb3ab8d68c49d0df8222fe0d11d628e65192790" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -3611,9 +3633,9 @@ dependencies = [ "rustls-webpki 0.102.8", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.4", "tokio-socks", "tokio-util", "tower 0.5.2", @@ -3639,7 +3661,7 @@ dependencies = [ "serde", "serde-value", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "winapi", ] @@ -3665,7 +3687,7 @@ dependencies = [ "rand 0.8.5", "rayon", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "winapi", "windows-sys 0.59.0", ] @@ -3704,7 +3726,7 @@ dependencies = [ "scopeguard", "serde", "smallvec", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-util", ] @@ -3724,7 +3746,7 @@ dependencies = [ "log", "once_cell", "os_pipe", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "pin-project", "rand 0.8.5", "tokio", @@ -3764,7 +3786,7 @@ dependencies = [ "rand 0.8.5", "rusqlite", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "url", ] @@ -3777,7 +3799,7 @@ dependencies = [ "deno_semver", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -3804,7 +3826,7 @@ dependencies = [ "libloading 0.7.4", "log", "napi_sym", - "thiserror 2.0.16", + "thiserror 2.0.17", "windows-sys 0.59.0", ] @@ -3838,7 +3860,7 @@ dependencies = [ "rustls-tokio-stream", "serde", "socket2 0.5.10", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", ] @@ -3925,7 +3947,7 @@ dependencies = [ "spki", "stable_deref_trait", "sys_traits", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-eld", "url", @@ -3953,7 +3975,7 @@ dependencies = [ "monch", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "url", ] @@ -3970,8 +3992,8 @@ dependencies = [ "stringcase", "strum 0.25.0", "strum_macros 0.25.3", - "syn 2.0.106", - "thiserror 2.0.16", + "syn 2.0.109", + "thiserror 2.0.17", ] [[package]] @@ -3992,7 +4014,7 @@ dependencies = [ "serde", "signal-hook", "signal-hook-registry", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "winapi", ] @@ -4011,7 +4033,7 @@ dependencies = [ "serde", "serde_json", "sys_traits", - "thiserror 2.0.16", + "thiserror 2.0.17", "url", ] @@ -4024,7 +4046,7 @@ dependencies = [ "deno_error", "percent-encoding", "sys_traits", - "thiserror 2.0.16", + "thiserror 2.0.17", "url", ] @@ -4045,7 +4067,7 @@ dependencies = [ "once_cell", "percent-encoding", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "which 6.0.3", "winapi", ] @@ -4072,7 +4094,7 @@ dependencies = [ "serde", "simd-json", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "which 6.0.3", "winapi", @@ -4104,9 +4126,9 @@ dependencies = [ "log", "node_resolver", "once_cell", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "sys_traits", - "thiserror 2.0.16", + "thiserror 2.0.17", "url", ] @@ -4171,7 +4193,7 @@ dependencies = [ "serde", "sys_traits", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-metrics", "twox-hash 1.6.3", @@ -4194,7 +4216,7 @@ dependencies = [ "monch", "once_cell", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "url", ] @@ -4221,7 +4243,7 @@ dependencies = [ "opentelemetry_sdk", "pin-project", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", ] @@ -4249,7 +4271,7 @@ dependencies = [ "rustls-tokio-stream", "rustls-webpki 0.102.8", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "webpki-roots 0.26.11", ] @@ -4261,7 +4283,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6742a724e8becb372a74c650a1aefb8924a5b8107f7d75b3848763ea24b27a87" dependencies = [ "futures-util", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "tokio", ] @@ -4273,7 +4295,7 @@ checksum = "d79e743ad841f7826d46c6944580f5ba665fe9ab4c31a68c4eed8b5a78225da3" dependencies = [ "deno_core", "deno_error", - "thiserror 2.0.16", + "thiserror 2.0.17", "urlpattern", ] @@ -4293,7 +4315,7 @@ dependencies = [ "flate2", "futures", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "uuid", ] @@ -4308,7 +4330,7 @@ dependencies = [ "deno_error", "raw-window-handle", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "wgpu-core", "wgpu-types", @@ -4344,7 +4366,7 @@ dependencies = [ "once_cell", "rustls-tokio-stream", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", ] @@ -4358,7 +4380,7 @@ dependencies = [ "deno_error", "deno_web", "rusqlite", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -4406,7 +4428,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-util", "url", @@ -4431,7 +4453,7 @@ dependencies = [ "rand 0.8.5", "rusqlite", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "uuid", @@ -4472,7 +4494,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -4493,7 +4515,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -4537,7 +4559,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -4557,7 +4579,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -4661,7 +4683,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.61.0", + "windows-sys 0.61.2", ] [[package]] @@ -4683,7 +4705,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -4718,14 +4740,14 @@ checksum = "788160fb30de9cdd857af31c6a2675904b16ece8fc2737b2c7127ba368c9d0f4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] name = "document-features" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" dependencies = [ "litrs", ] @@ -4809,13 +4831,20 @@ dependencies = [ [[package]] name = "dyn-stack" -version = "0.13.0" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490bd48eb68fffcfed519b4edbfd82c69cbe741d175b84f0e0cbe8c57cbe0bdd" +checksum = "1c4713e43e2886ba72b8271aa66c93d722116acf7a75555cce11dcde84388fe8" dependencies = [ "bytemuck", + "dyn-stack-macros", ] +[[package]] +name = "dyn-stack-macros" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9" + [[package]] name = "dynasm" version = "1.2.3" @@ -4921,7 +4950,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -4987,27 +5016,27 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] name = "enum-ordinalize" -version = "4.3.0" +version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea0dcfa4e54eeb516fe454635a95753ddd39acda650ce703031c6973e315dd5" +checksum = "4a1091a7bb1f8f2c4b28f1fe2cef4980ca2d410a3d727d67ecc3178c9b0800f0" dependencies = [ "enum-ordinalize-derive", ] [[package]] name = "enum-ordinalize-derive" -version = "4.3.1" +version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" +checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -5027,7 +5056,7 @@ checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -5053,7 +5082,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.0", + "windows-sys 0.61.2", ] [[package]] @@ -5141,7 +5170,7 @@ checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" dependencies = [ "bit-set 0.5.3", "regex-automata", - "regex-syntax 0.8.6", + "regex-syntax 0.8.8", ] [[package]] @@ -5152,7 +5181,7 @@ checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" dependencies = [ "bit-set 0.8.0", "regex-automata", - "regex-syntax 0.8.6", + "regex-syntax 0.8.8", ] [[package]] @@ -5252,9 +5281,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.1" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" +checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" [[package]] name = "fixedbitset" @@ -5264,9 +5293,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flatbuffers" -version = "25.2.10" +version = "25.9.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" +checksum = "09b6620799e7340ebd9968d2e0708eb82cf1971e9a16821e2091b6d6e475eed5" dependencies = [ "bitflags 2.9.4", "rustc_version 0.4.1", @@ -5274,9 +5303,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.2" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" dependencies = [ "crc32fast", "libz-rs-sys", @@ -5344,7 +5373,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -5382,7 +5411,7 @@ checksum = "32016f1242eb82af5474752d00fd8ebcd9004bd69b462b1c91de833972d08ed4" dependencies = [ "proc-macro2", "swc_macros_common", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -5487,7 +5516,7 @@ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" dependencies = [ "futures-core", "lock_api", - "parking_lot 0.12.4", + "parking_lot 0.12.5", ] [[package]] @@ -5517,7 +5546,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -5601,7 +5630,7 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab96b703d31950f1aeddded248bc95543c9efc7ac9c4a21fda8703a83ee35451" dependencies = [ - "dyn-stack 0.13.0", + "dyn-stack 0.13.2", "gemm-c32 0.18.2", "gemm-c64 0.18.2", "gemm-common 0.18.2", @@ -5636,7 +5665,7 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6db9fd9f40421d00eea9dd0770045a5603b8d684654816637732463f4073847" dependencies = [ - "dyn-stack 0.13.0", + "dyn-stack 0.13.2", "gemm-common 0.18.2", "num-complex", "num-traits", @@ -5666,7 +5695,7 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfcad8a3d35a43758330b635d02edad980c1e143dc2f21e6fd25f9e4eada8edf" dependencies = [ - "dyn-stack 0.13.0", + "dyn-stack 0.13.2", "gemm-common 0.18.2", "num-complex", "num-traits", @@ -5702,7 +5731,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a352d4a69cbe938b9e2a9cb7a3a63b7e72f9349174a2752a558a8a563510d0f3" dependencies = [ "bytemuck", - "dyn-stack 0.13.0", + "dyn-stack 0.13.2", "half", "libm", "num-complex", @@ -5740,7 +5769,7 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cff95ae3259432f3c3410eaa919033cd03791d81cebd18018393dc147952e109" dependencies = [ - "dyn-stack 0.13.0", + "dyn-stack 0.13.2", "gemm-common 0.18.2", "gemm-f32 0.18.2", "half", @@ -5773,7 +5802,7 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc8d3d4385393304f407392f754cd2dc4b315d05063f62cf09f47b58de276864" dependencies = [ - "dyn-stack 0.13.0", + "dyn-stack 0.13.2", "gemm-common 0.18.2", "num-complex", "num-traits", @@ -5803,7 +5832,7 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35b2a4f76ce4b8b16eadc11ccf2e083252d8237c1b589558a49b0183545015bd" dependencies = [ - "dyn-stack 0.13.0", + "dyn-stack 0.13.2", "gemm-common 0.18.2", "num-complex", "num-traits", @@ -5812,25 +5841,11 @@ dependencies = [ "seq-macro", ] -[[package]] -name = "generator" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "605183a538e3e2a9c1038635cc5c2d194e2ee8fd0d1b66b8349fad7dbacce5a2" -dependencies = [ - "cc", - "cfg-if", - "libc", - "log", - "rustversion", - "windows 0.61.3", -] - [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", @@ -5853,7 +5868,7 @@ version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" dependencies = [ - "unicode-width 0.2.1", + "unicode-width 0.2.2", ] [[package]] @@ -5865,21 +5880,21 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "js-sys", "libc", "r-efi", - "wasi 0.14.5+wasi-0.2.4", + "wasip2", "wasm-bindgen", ] @@ -5895,9 +5910,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.1" +version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" [[package]] name = "git-version" @@ -5916,7 +5931,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -5938,15 +5953,15 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "globset" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5" +checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3" dependencies = [ "aho-corasick", "bstr", "log", "regex-automata", - "regex-syntax 0.8.6", + "regex-syntax 0.8.8", ] [[package]] @@ -5994,7 +6009,7 @@ dependencies = [ "google-cloud-token", "home", "jsonwebtoken 9.3.1", - "reqwest 0.12.23", + "reqwest 0.12.24", "serde", "serde_json", "thiserror 1.0.69", @@ -6037,7 +6052,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d901aeb453fd80e51d64df4ee005014f6cf39f2d736dd64f7239c132d9d39a6a" dependencies = [ - "reqwest 0.12.23", + "reqwest 0.12.24", "thiserror 1.0.69", "tokio", ] @@ -6181,9 +6196,9 @@ dependencies = [ [[package]] name = "half" -version = "2.6.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "bytemuck", "cfg-if", @@ -6191,6 +6206,7 @@ dependencies = [ "num-traits", "rand 0.9.0", "rand_distr 0.5.1", + "zerocopy", ] [[package]] @@ -6233,6 +6249,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" + [[package]] name = "hashify" version = "0.2.7" @@ -6241,7 +6263,7 @@ checksum = "149e3ea90eb5a26ad354cfe3cb7f7401b9329032d0235f2687d03a35f30e5d4c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -6345,10 +6367,10 @@ dependencies = [ "native-tls", "num_cpus", "rand 0.9.0", - "reqwest 0.12.23", + "reqwest 0.12.24", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "ureq", "windows-sys 0.60.2", @@ -6373,7 +6395,7 @@ dependencies = [ "once_cell", "rand 0.9.0", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "tinyvec", "tokio", "tracing", @@ -6392,12 +6414,12 @@ dependencies = [ "ipconfig", "moka", "once_cell", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "rand 0.9.0", "resolv-conf", "serde", "smallvec", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", ] @@ -6433,11 +6455,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6541,9 +6563,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" @@ -6608,7 +6630,7 @@ dependencies = [ "pin-project-lite", "rustls-native-certs 0.7.3", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.4", "tower-service", ] @@ -6654,12 +6676,12 @@ dependencies = [ "hyper-util", "log", "rustls 0.23.29", - "rustls-native-certs 0.8.1", + "rustls-native-certs 0.8.2", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.4", "tower-service", - "webpki-roots 1.0.2", + "webpki-roots 1.0.4", ] [[package]] @@ -6706,9 +6728,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" dependencies = [ "base64 0.22.1", "bytes", @@ -6722,7 +6744,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.0", + "socket2 0.6.1", "system-configuration 0.6.1", "tokio", "tower-service", @@ -6756,9 +6778,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.63" +version = "0.1.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -6766,7 +6788,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.61.2", + "windows-core 0.62.2", ] [[package]] @@ -6780,22 +6802,22 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" dependencies = [ "displaydoc", "potential_utf", - "yoke 0.8.0", + "yoke 0.8.1", "zerofrom", "zerovec", ] [[package]] name = "icu_locale_core" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" dependencies = [ "displaydoc", "litemap", @@ -6806,11 +6828,10 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" dependencies = [ - "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", @@ -6821,44 +6842,40 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" dependencies = [ - "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", "icu_provider", - "potential_utf", "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" +checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" [[package]] name = "icu_provider" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" dependencies = [ "displaydoc", "icu_locale_core", - "stable_deref_trait", - "tinystr", "writeable", - "yoke 0.8.0", + "yoke 0.8.1", "zerofrom", "zerotrie", "zerovec", @@ -6899,9 +6916,9 @@ checksum = "cd62e6b5e86ea8eeeb8db1de02880a6abc01a397b2ebb64b5d74ac255318f5cb" [[package]] name = "ignore" -version = "0.4.23" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b" +checksum = "d3d782a365a015e0f5c04902246139249abf769125006fbe7649e2ee88169b4a" dependencies = [ "crossbeam-deque", "globset", @@ -6939,7 +6956,7 @@ dependencies = [ "percent-encoding", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "url", ] @@ -6974,7 +6991,7 @@ dependencies = [ "console", "number_prefix", "portable-atomic", - "unicode-width 0.2.1", + "unicode-width 0.2.2", "web-time", ] @@ -7037,9 +7054,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" +checksum = "fdd7bddefd0a8833b88a4b68f90dae22c7450d11b354198baee3874fd811b344" dependencies = [ "bitflags 2.9.4", "cfg-if", @@ -7075,9 +7092,9 @@ dependencies = [ [[package]] name = "iri-string" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" dependencies = [ "memchr", "serde", @@ -7092,7 +7109,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -7103,9 +7120,9 @@ checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" [[package]] name = "is_terminal_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itertools" @@ -7170,15 +7187,15 @@ version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "libc", ] [[package]] name = "js-sys" -version = "0.3.78" +version = "0.3.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0b063578492ceec17683ef2f8c5e89121fbd0b172cbc280635ab7567db2738" +checksum = "852f13bec5eba4ba9afbeb93fd7c13fe56147f055939ae21c43a29a0ecb2702e" dependencies = [ "once_cell", "wasm-bindgen", @@ -7186,9 +7203,9 @@ dependencies = [ [[package]] name = "json-patch" -version = "4.0.0" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "159294d661a039f7644cea7e4d844e6b25aaf71c1ffe9d73a96d768c24b0faf4" +checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" dependencies = [ "jsonptr", "serde", @@ -7215,7 +7232,7 @@ dependencies = [ "pest_derive", "regex", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -7318,7 +7335,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" dependencies = [ "libc", - "libloading 0.8.8", + "libloading 0.8.9", "pkg-config", ] @@ -7405,7 +7422,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-util", "tower 0.5.2", @@ -7429,7 +7446,7 @@ dependencies = [ "serde", "serde-value", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -7443,7 +7460,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -7463,11 +7480,11 @@ dependencies = [ "json-patch", "k8s-openapi", "kube-client", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "pin-project", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-util", "tracing", @@ -7481,9 +7498,9 @@ checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" [[package]] name = "lazy-regex" -version = "3.4.1" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60c7310b93682b36b98fa7ea4de998d3463ccbebd94d935d6b48ba5b6ffa7126" +checksum = "191898e17ddee19e60bccb3945aa02339e81edd4a8c50e21fd4d48cdecda7b29" dependencies = [ "lazy-regex-proc_macros", "once_cell", @@ -7492,14 +7509,14 @@ dependencies = [ [[package]] name = "lazy-regex-proc_macros" -version = "3.4.1" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ba01db5ef81e17eb10a5e0f2109d1b3a3e29bac3070fdbd7d156bf7dbd206a1" +checksum = "c35dc8b0da83d1a9507e12122c80dea71a9c7c613014347392483a83ea593e04" dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -7525,9 +7542,9 @@ checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" [[package]] name = "lexical-core" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -7538,60 +7555,53 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" dependencies = [ "lexical-parse-integer", "lexical-util", - "static_assertions", ] [[package]] name = "lexical-parse-integer" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" dependencies = [ "lexical-util", - "static_assertions", ] [[package]] name = "lexical-util" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" -dependencies = [ - "static_assertions", -] +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" [[package]] name = "lexical-write-float" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" dependencies = [ "lexical-util", "lexical-write-integer", - "static_assertions", ] [[package]] name = "lexical-write-integer" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" dependencies = [ "lexical-util", - "static_assertions", ] [[package]] name = "libc" -version = "0.2.175" +version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" [[package]] name = "libffi" @@ -7624,12 +7634,12 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "windows-targets 0.53.3", + "windows-link 0.2.1", ] [[package]] @@ -7640,24 +7650,24 @@ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libproc" -version = "0.14.10" +version = "0.14.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78a09b56be5adbcad5aa1197371688dc6bb249a26da3bca2011ee2fb987ebfb" +checksum = "a54ad7278b8bc5301d5ffd2a94251c004feb971feba96c971ea4063645990757" dependencies = [ - "bindgen 0.70.1", + "bindgen 0.72.1", "errno", "libc", ] [[package]] name = "libredox" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" +checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ "bitflags 2.9.4", "libc", - "redox_syscall 0.5.17", + "redox_syscall 0.5.18", ] [[package]] @@ -7733,23 +7743,22 @@ checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "litrs" -version = "0.4.2" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e54036fe321fd421e10d732f155734c4e4afd610dd556d9a82833ab3ee0bed" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "lock_api" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" dependencies = [ - "autocfg", "scopeguard", ] @@ -7769,19 +7778,6 @@ dependencies = [ "prost-types", ] -[[package]] -name = "loom" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" -dependencies = [ - "cfg-if", - "generator", - "scoped-tls", - "tracing", - "tracing-subscriber", -] - [[package]] name = "lru" version = "0.12.5" @@ -7910,7 +7906,7 @@ dependencies = [ "rustls-pki-types", "smtp-proto", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.4", "webpki-roots 0.26.11", ] @@ -8049,9 +8045,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "memmap2" @@ -8064,9 +8060,9 @@ dependencies = [ [[package]] name = "memmap2" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843a98750cd611cc2965a8213b53b43e715f13c37a9e096c6408e69990961db7" +checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" dependencies = [ "libc", "stable_deref_trait", @@ -8127,7 +8123,7 @@ checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -8189,37 +8185,36 @@ checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "log", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "windows-sys 0.48.0", ] [[package]] name = "mio" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" dependencies = [ "libc", - "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.59.0", + "wasi", + "windows-sys 0.61.2", ] [[package]] name = "moka" -version = "0.12.10" +version = "0.12.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" +checksum = "8261cd88c312e0004c1d51baad2980c66528dfdb2bee62003e643a4d8f86b077" dependencies = [ "crossbeam-channel", "crossbeam-epoch", "crossbeam-utils", - "loom", - "parking_lot 0.12.4", + "equivalent", + "parking_lot 0.12.5", "portable-atomic", "rustc_version 0.4.1", "smallvec", "tagptr", - "thiserror 1.0.69", "uuid", ] @@ -8247,7 +8242,7 @@ checksum = "c402a4092d5e204f32c9e155431046831fa712637043c58cb73bc6bc6c9663b5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -8292,9 +8287,9 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", "termcolor", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -8319,7 +8314,7 @@ dependencies = [ "serde", "serde_json", "socket2 0.5.10", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-native-tls", "tokio-util", @@ -8340,7 +8335,7 @@ dependencies = [ "bytes", "crc32fast", "flate2", - "getrandom 0.3.3", + "getrandom 0.3.4", "mysql-common-derive", "num-bigint", "num-traits", @@ -8351,7 +8346,7 @@ dependencies = [ "serde_json", "sha1", "sha2 0.10.9", - "thiserror 2.0.16", + "thiserror 2.0.17", "uuid", ] @@ -8395,7 +8390,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -8521,7 +8516,7 @@ dependencies = [ "serde", "serde_json", "sys_traits", - "thiserror 2.0.16", + "thiserror 2.0.17", "url", ] @@ -8584,11 +8579,11 @@ dependencies = [ [[package]] name = "nu-ansi-term" -version = "0.50.1" +version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -8601,7 +8596,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -8678,7 +8673,7 @@ dependencies = [ "num-format", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "typetag", "windows-sys 0.48.0", ] @@ -8758,11 +8753,10 @@ dependencies = [ [[package]] name = "num-bigint-dig" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +checksum = "82c79c15c05d4bf82b6f5ef163104cc81a760d8e874d38ac50ab67c8877b647b" dependencies = [ - "byteorder", "lazy_static", "libm", "num-integer", @@ -8853,9 +8847,9 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a973b4e44ce6cad84ce69d797acf9a044532e4184c4f267913d1b546a0727b7a" +checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c" dependencies = [ "num_enum_derive", "rustversion", @@ -8863,14 +8857,14 @@ dependencies = [ [[package]] name = "num_enum_derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e878c846a8abae00dd069496dbe8751b16ac1c3d6bd2a7283a938e8228f90d" +checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -8890,7 +8884,7 @@ dependencies = [ "getrandom 0.2.16", "http 1.3.1", "rand 0.8.5", - "reqwest 0.12.23", + "reqwest 0.12.24", "serde", "serde_json", "serde_path_to_error", @@ -8910,9 +8904,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.7" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ "memchr", ] @@ -8935,17 +8938,17 @@ dependencies = [ "hyper 1.7.0", "itertools 0.14.0", "md-5 0.10.6", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "percent-encoding", "quick-xml 0.37.5", "rand 0.9.0", - "reqwest 0.12.23", + "reqwest 0.12.24", "ring 0.17.14", "rustls-pemfile 2.2.0", "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", "url", @@ -8986,9 +8989,9 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "once_cell_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "oneshot" @@ -9068,9 +9071,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.73" +version = "0.10.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" +checksum = "24ad14dd45412269e1a30f52ad8f0664f0f4f4a89ee8fe28c3b3527021ebb654" dependencies = [ "bitflags 2.9.4", "cfg-if", @@ -9089,7 +9092,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -9100,18 +9103,18 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-src" -version = "300.5.2+3.5.2" +version = "300.5.4+3.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d270b79e2926f5150189d475bc7e9d2c69f9c4697b185fa917d5a32b792d21b4" +checksum = "a507b3792995dae9b0df8a1c1e3771e8418b7c2d9f0baeba32e6fe8b06c7cb72" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.109" +version = "0.9.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" +checksum = "0a9f0075ba3c21b09f8e8b2026584b1d18d49388648f2fbbf3c97ea8deced8e2" dependencies = [ "cc", "libc", @@ -9294,9 +9297,9 @@ dependencies = [ [[package]] name = "owo-colors" -version = "4.2.2" +version = "4.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48dd4f4a2c8405440fd0462561f0e5806bd0f77e86f51c761481bdd4018b545e" +checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52" [[package]] name = "p224" @@ -9367,12 +9370,12 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", - "parking_lot_core 0.9.11", + "parking_lot_core 0.9.12", ] [[package]] @@ -9391,15 +9394,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.11" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.17", + "redox_syscall 0.5.18", "smallvec", - "windows-targets 0.52.6", + "windows-link 0.2.1", ] [[package]] @@ -9513,7 +9516,7 @@ checksum = "31095ca1f396e3de32745f42b20deef7bc09077f918b085307e8eab6ddd8fb9c" dependencies = [ "once_cell", "serde", - "unicode-width 0.2.1", + "unicode-width 0.2.2", "unscanny", ] @@ -9525,20 +9528,19 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.1" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" +checksum = "989e7521a040efde50c3ab6bbadafbe15ab6dc042686926be59ac35d74607df4" dependencies = [ "memchr", - "thiserror 2.0.16", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.8.1" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb056d9e8ea77922845ec74a1c4e8fb17e7c218cc4fc11a15c5d25e189aa40bc" +checksum = "187da9a3030dbafabbbfb20cb323b976dc7b7ce91fcd84f2f74d6e31d378e2de" dependencies = [ "pest", "pest_generator", @@ -9546,22 +9548,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.1" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e404e638f781eb3202dc82db6760c8ae8a1eeef7fb3fa8264b2ef280504966" +checksum = "49b401d98f5757ebe97a26085998d6c0eecec4995cad6ab7fc30ffdf4b052843" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] name = "pest_meta" -version = "2.8.1" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd1101f170f5903fde0914f899bb503d9ff5271d7ba76bbb70bea63690cc0d5" +checksum = "72f27a2cfee9f9039c4d86faa5af122a0ac3851441a34865b8a043b46be0065a" dependencies = [ "pest", "sha2 0.10.9", @@ -9635,7 +9637,7 @@ dependencies = [ "phf_shared 0.11.3", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -9685,7 +9687,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -9817,9 +9819,9 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76ff0abab4a9b844b93ef7b81f1efc0a366062aaef2cd702c76256b5dc075c54" +checksum = "fbef655056b916eb868048276cfd5d6a7dea4f81560dfd047f97c8c6fe3fcfd4" dependencies = [ "base64 0.22.1", "byteorder", @@ -9854,7 +9856,7 @@ dependencies = [ "bytes", "chrono", "fallible-iterator 0.2.0", - "postgres-protocol 0.6.8", + "postgres-protocol 0.6.9", "serde", "serde_json", "uuid", @@ -9862,9 +9864,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" dependencies = [ "zerovec", ] @@ -9897,7 +9899,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -9911,11 +9913,11 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ - "toml_edit 0.22.27", + "toml_edit 0.23.4", ] [[package]] @@ -9961,7 +9963,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -9972,7 +9974,7 @@ checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f" dependencies = [ "proc-macro-rules-macros", "proc-macro2", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -9984,14 +9986,14 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] name = "proc-macro2" -version = "1.0.101" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" dependencies = [ "unicode-ident", ] @@ -10065,8 +10067,8 @@ dependencies = [ "fnv", "lazy_static", "memchr", - "parking_lot 0.12.4", - "thiserror 2.0.16", + "parking_lot 0.12.5", + "thiserror 2.0.17", ] [[package]] @@ -10095,7 +10097,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.106", + "syn 2.0.109", "tempfile", ] @@ -10109,7 +10111,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -10123,10 +10125,11 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.26" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e944464ec8536cd1beb0bbfd96987eb5e3b72f2ecdafdc5c769a37f1fa2ae1f" +checksum = "d11f2fedc3b7dafdc2851bc52f277377c5473d378859be234bc7ebb593144d01" dependencies = [ + "ar_archive_writer", "cc", ] @@ -10190,9 +10193,9 @@ dependencies = [ [[package]] name = "pure-rust-locales" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1190fd18ae6ce9e137184f207593877e70f39b015040156b1e05081cdfe3733a" +checksum = "869675ad2d7541aea90c6d88c81f46a7f4ea9af8cd0395d38f11a95126998a0d" [[package]] name = "pwd" @@ -10226,14 +10229,14 @@ dependencies = [ [[package]] name = "quick_cache" -version = "0.6.16" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ad6644cb07b7f3488b9f3d2fde3b4c0a7fa367cafefb39dff93a659f76eb786" +checksum = "7ada44a88ef953a3294f6eb55d2007ba44646015e18613d2f213016379203ef3" dependencies = [ "ahash 0.8.12", "equivalent", - "hashbrown 0.15.5", - "parking_lot 0.12.4", + "hashbrown 0.16.0", + "parking_lot 0.12.5", ] [[package]] @@ -10249,8 +10252,8 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.29", - "socket2 0.6.0", - "thiserror 2.0.16", + "socket2 0.6.1", + "thiserror 2.0.17", "tokio", "tracing", "web-time", @@ -10263,7 +10266,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ "bytes", - "getrandom 0.3.3", + "getrandom 0.3.4", "lru-slab", "rand 0.9.0", "ring 0.17.14", @@ -10271,7 +10274,7 @@ dependencies = [ "rustls 0.23.29", "rustls-pki-types", "slab", - "thiserror 2.0.16", + "thiserror 2.0.17", "tinyvec", "tracing", "web-time", @@ -10286,16 +10289,16 @@ dependencies = [ "cfg_aliases 0.2.1", "libc", "once_cell", - "socket2 0.6.0", + "socket2 0.6.1", "tracing", "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.40" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" dependencies = [ "proc-macro2", ] @@ -10379,7 +10382,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", ] [[package]] @@ -10518,7 +10521,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -10541,9 +10544,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.17" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ "bitflags 2.9.4", ] @@ -10567,57 +10570,57 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.16", "libredox", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "ref-cast" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] name = "regex" -version = "1.11.2" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax 0.8.6", + "regex-syntax 0.8.8", ] [[package]] name = "regex-automata" -version = "0.4.10" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.6", + "regex-syntax 0.8.8", ] [[package]] name = "regex-lite" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943f41321c63ef1c92fd763bfe054d2668f7f225a5c29f0105903dc2fc04ba30" +checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" [[package]] name = "regex-syntax" @@ -10627,9 +10630,9 @@ checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "regex-syntax" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "rend" @@ -10684,9 +10687,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.23" +version = "0.12.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" +checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" dependencies = [ "async-compression", "base64 0.22.1", @@ -10711,7 +10714,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.29", - "rustls-native-certs 0.8.1", + "rustls-native-certs 0.8.2", "rustls-pki-types", "serde", "serde_json", @@ -10719,7 +10722,7 @@ dependencies = [ "sync_wrapper 1.0.2", "tokio", "tokio-native-tls", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.4", "tokio-util", "tower 0.5.2", "tower-http", @@ -10729,7 +10732,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.2", + "webpki-roots 1.0.4", ] [[package]] @@ -10741,7 +10744,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.3.1", - "reqwest 0.12.23", + "reqwest 0.12.24", "serde", "thiserror 1.0.69", "tower-service", @@ -10760,7 +10763,7 @@ dependencies = [ "http 1.3.1", "hyper 1.7.0", "parking_lot 0.11.2", - "reqwest 0.12.23", + "reqwest 0.12.24", "reqwest-middleware", "retry-policies", "thiserror 1.0.69", @@ -10771,9 +10774,9 @@ dependencies = [ [[package]] name = "resolv-conf" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95325155c684b1c89f7765e30bc1c42e4a6da51ca513615660cb8a62ef9a88e3" +checksum = "6b3789b30bd25ba102de4beabd95d21ac45b69b1be7d14522bab988c526d6799" [[package]] name = "retry-policies" @@ -10863,9 +10866,9 @@ dependencies = [ [[package]] name = "rmcp" -version = "0.2.1" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37f2048a81a7ff7e8ef6bc5abced70c3d9114c8f03d85d7aaaafd9fd04f12e9e" +checksum = "e5947688160b56fb6c827e3c20a72c90392a1d7e9dec74749197aa1780ac42ca" dependencies = [ "base64 0.22.1", "bytes", @@ -10877,12 +10880,13 @@ dependencies = [ "paste", "pin-project-lite", "rand 0.9.0", + "reqwest 0.12.24", "rmcp-macros", - "schemars 0.8.22", + "schemars 1.1.0", "serde", "serde_json", "sse-stream", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tokio-util", @@ -10893,15 +10897,15 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "0.2.1" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72398e694b9f6dbb5de960cf158c8699e6a1854cb5bbaac7de0646b2005763c4" +checksum = "01263441d3f8635c628e33856c468b96ebbce1af2d3699ea712ca71432d4ee7a" dependencies = [ - "darling 0.20.11", + "darling 0.21.3", "proc-macro2", "quote", "serde_json", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -10991,7 +10995,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.106", + "syn 2.0.109", "walkdir", ] @@ -11017,9 +11021,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.37.2" +version = "1.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b203a6425500a03e0919c42d3c47caca51e79f1132046626d2c8871c5092035d" +checksum = "35affe401787a9bd846712274d97654355d21b2a2c092a3139aabe31e9022282" dependencies = [ "arrayvec", "borsh", @@ -11065,7 +11069,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.26", + "semver 1.0.27", ] [[package]] @@ -11100,7 +11104,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.0", + "windows-sys 0.61.2", ] [[package]] @@ -11140,7 +11144,7 @@ dependencies = [ "once_cell", "ring 0.17.14", "rustls-pki-types", - "rustls-webpki 0.103.5", + "rustls-webpki 0.103.8", "subtle", "zeroize", ] @@ -11172,14 +11176,14 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.4.0", + "security-framework 3.5.1", ] [[package]] @@ -11202,9 +11206,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" dependencies = [ "web-time", "zeroize", @@ -11245,9 +11249,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.5" +version = "0.103.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a37813727b78798e53c2bec3f5e8fe12a6d6f8389bf9ca7802add4c9905ad8" +checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" dependencies = [ "aws-lc-rs", "ring 0.17.14", @@ -11429,7 +11433,7 @@ version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys 0.61.0", + "windows-sys 0.61.2", ] [[package]] @@ -11438,9 +11442,8 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" dependencies = [ - "chrono", "dyn-clone", - "schemars_derive", + "schemars_derive 0.8.22", "serde", "serde_json", ] @@ -11459,12 +11462,14 @@ dependencies = [ [[package]] name = "schemars" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0" +checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" dependencies = [ + "chrono", "dyn-clone", "ref-cast", + "schemars_derive 1.1.0", "serde", "serde_json", ] @@ -11478,7 +11483,19 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.106", + "syn 2.0.109", +] + +[[package]] +name = "schemars_derive" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301858a4023d78debd2353c7426dc486001bddc91ae31a76fb1f55132f7e2633" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.109", ] [[package]] @@ -11560,9 +11577,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.4.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b369d18893388b345804dc0007963c99b7d665ae71d275812d828c6f089640" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" dependencies = [ "bitflags 2.9.4", "core-foundation 0.10.1", @@ -11592,9 +11609,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "semver-parser" @@ -11667,7 +11684,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -11678,7 +11695,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -11730,7 +11747,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -11764,15 +11781,15 @@ dependencies = [ "num-bigint", "serde", "smallvec", - "thiserror 2.0.16", + "thiserror 2.0.17", "v8", ] [[package]] name = "serde_with" -version = "3.14.0" +version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" +checksum = "c522100790450cf78eeac1507263d0a350d4d5b30df0c8e1fe051a10c22b376e" dependencies = [ "base64 0.22.1", "chrono", @@ -11780,7 +11797,7 @@ dependencies = [ "indexmap 1.9.3", "indexmap 2.11.1", "schemars 0.9.0", - "schemars 1.0.4", + "schemars 1.1.0", "serde", "serde_derive", "serde_json", @@ -11790,14 +11807,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.14.0" +version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" +checksum = "327ada00f7d64abaac1e55a6911e90cf665aa051b9a561c7006c157f4633135e" dependencies = [ - "darling 0.20.11", + "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -11992,7 +12009,7 @@ checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.16", + "thiserror 2.0.17", "time", ] @@ -12091,12 +12108,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -12228,7 +12245,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -12274,7 +12291,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "smallvec", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tracing", @@ -12293,7 +12310,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -12316,7 +12333,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.106", + "syn 2.0.109", "tokio", "url", ] @@ -12360,7 +12377,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", "uuid", "whoami", @@ -12401,7 +12418,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", "uuid", "whoami", @@ -12427,7 +12444,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", "url", "uuid", @@ -12448,15 +12465,15 @@ dependencies = [ [[package]] name = "stable_deref_trait" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cddb07e32ddb770749da91081d8d0ac3a16f1a569a18b20348cd371f5dead06b" +checksum = "e1f8b29fb42aafcea4edeeb6b2f2d7ecd0d969c48b4cf0d2e64aafc471dd6e59" dependencies = [ "cc", "cfg-if", @@ -12480,7 +12497,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -12549,7 +12566,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -12561,7 +12578,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -12680,7 +12697,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -12729,7 +12746,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -12814,7 +12831,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -12921,7 +12938,7 @@ checksum = "63db0adcff29d220c3d151c5b25c0eabe7e32dd936212b84cdaa1392e3130497" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -12932,7 +12949,7 @@ checksum = "f486687bfb7b5c560868f69ed2d458b880cebc9babebcb67e49f31b55c5bf847" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -12955,7 +12972,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -12971,9 +12988,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.106" +version = "2.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +checksum = "2f17c7e013e88258aa9543dcbe81aca68a667a9ac37cd69c9fbc07858bfe0e2f" dependencies = [ "proc-macro2", "quote", @@ -13015,7 +13032,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -13167,7 +13184,7 @@ dependencies = [ "lru 0.12.5", "lz4_flex", "measure_time", - "memmap2 0.9.8", + "memmap2 0.9.9", "once_cell", "oneshot", "rayon", @@ -13186,7 +13203,7 @@ dependencies = [ "tantivy-stacker", "tantivy-tokenizer-api", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "time", "uuid", "winapi", @@ -13234,7 +13251,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" dependencies = [ "byteorder", - "regex-syntax 0.8.6", + "regex-syntax 0.8.8", "utf8-ranges", ] @@ -13298,15 +13315,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84fa4d11fadde498443cca10fd3ac23c951f0dc59e080e9f4b93d4df4e4eea53" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ "fastrand", - "getrandom 0.3.3", + "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.0", + "windows-sys 0.61.2", ] [[package]] @@ -13344,7 +13361,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" dependencies = [ "unicode-linebreak", - "unicode-width 0.2.1", + "unicode-width 0.2.2", ] [[package]] @@ -13358,11 +13375,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl 2.0.16", + "thiserror-impl 2.0.17", ] [[package]] @@ -13373,18 +13390,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] name = "thiserror-impl" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -13480,11 +13497,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.43" +version = "0.3.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bde6f1ec10e72d583d91623c939f623002284ef622b87de38cfd546cbf2031" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" dependencies = [ "deranged", + "itoa", "num-conv", "powerfmt", "serde", @@ -13519,9 +13537,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" dependencies = [ "displaydoc", "zerovec", @@ -13600,8 +13618,8 @@ dependencies = [ "bytes", "io-uring", "libc", - "mio 1.0.4", - "parking_lot 0.12.4", + "mio 1.1.0", + "parking_lot 0.12.5", "pin-project-lite", "signal-hook-registry", "slab", @@ -13629,7 +13647,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -13666,7 +13684,7 @@ dependencies = [ "futures-channel", "futures-util", "log", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "percent-encoding", "phf 0.11.3", "pin-project-lite", @@ -13692,11 +13710,11 @@ dependencies = [ "futures-channel", "futures-util", "log", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "percent-encoding", "phf 0.11.3", "pin-project-lite", - "postgres-protocol 0.6.8", + "postgres-protocol 0.6.9", "postgres-types 0.2.9", "rand 0.9.0", "socket2 0.5.10", @@ -13738,9 +13756,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ "rustls 0.23.29", "tokio", @@ -13800,9 +13818,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.16" +version = "0.7.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" +checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" dependencies = [ "bytes", "futures-core", @@ -13829,10 +13847,10 @@ dependencies = [ "httparse", "rand 0.8.5", "ring 0.17.14", - "rustls-native-certs 0.8.1", + "rustls-native-certs 0.8.2", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.4", "tokio-util", ] @@ -13844,7 +13862,7 @@ checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" dependencies = [ "serde", "serde_spanned", - "toml_datetime", + "toml_datetime 0.6.11", "toml_edit 0.19.15", ] @@ -13857,6 +13875,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3" +dependencies = [ + "serde", +] + [[package]] name = "toml_edit" version = "0.19.15" @@ -13866,18 +13893,28 @@ dependencies = [ "indexmap 2.11.1", "serde", "serde_spanned", - "toml_datetime", + "toml_datetime 0.6.11", "winnow 0.5.40", ] [[package]] name = "toml_edit" -version = "0.22.27" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +checksum = "7211ff1b8f0d3adae1663b7da9ffe396eabe1ca25f0b0bee42b0da29a9ddce93" dependencies = [ "indexmap 2.11.1", - "toml_datetime", + "toml_datetime 0.7.0", + "toml_parser", + "winnow 0.7.13", +] + +[[package]] +name = "toml_parser" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" +dependencies = [ "winnow 0.7.13", ] @@ -13903,11 +13940,11 @@ dependencies = [ "percent-encoding", "pin-project", "prost", - "rustls-native-certs 0.8.1", + "rustls-native-certs 0.8.2", "rustls-pemfile 2.2.0", "socket2 0.5.10", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.4", "tokio-stream", "tower 0.4.13", "tower-layer", @@ -13964,7 +14001,7 @@ dependencies = [ "cookie 0.18.1", "futures-util", "http 1.3.1", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "pin-project-lite", "tower-layer", "tower-service", @@ -14040,7 +14077,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -14081,7 +14118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3beec919fbdf99d719de8eda6adae3281f8a5b71ae40431f44dc7423053d34" dependencies = [ "loki-api", - "reqwest 0.12.23", + "reqwest 0.12.24", "serde", "serde_json", "snap", @@ -14152,7 +14189,7 @@ checksum = "0203df02a3b6dd63575cc1d6e609edc2181c9a11867a271b25cfd2abff3ec5ca" dependencies = [ "cc", "regex", - "regex-syntax 0.8.6", + "regex-syntax 0.8.8", "tree-sitter-language", ] @@ -14194,9 +14231,9 @@ dependencies = [ [[package]] name = "triomphe" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8f7726da4807b58ea5c96fdc122f80702030edc33b35aff9190a51148ccc85" +checksum = "dd69c5aa8f924c7519d6372789a74eac5b94fb0f8fcf0d4a97eb0bfc3e785f39" dependencies = [ "serde", "stable_deref_trait", @@ -14268,15 +14305,15 @@ checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" [[package]] name = "typenum" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] name = "typetag" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f22b40dd7bfe8c14230cf9702081366421890435b2d625fa92b4acc4c3de6f" +checksum = "be2212c8a9b9bcfca32024de14998494cf9a5dfa59ea1b829de98bac374b86bf" dependencies = [ "erased-serde", "inventory", @@ -14287,13 +14324,13 @@ dependencies = [ [[package]] name = "typetag-impl" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35f5380909ffc31b4de4f4bdf96b877175a016aa2ca98cee39fcfd8c4d53d952" +checksum = "27a7a9b72ba121f6f1f6c3632b85604cac41aedb5ddc70accbebb6cac83de846" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -14310,8 +14347,8 @@ checksum = "90b70b37e9074642bc5f60bb23247fd072a84314ca9e71cdf8527593406a0dd3" dependencies = [ "gemm 0.18.2", "half", - "libloading 0.8.8", - "memmap2 0.9.8", + "libloading 0.8.9", + "memmap2 0.9.9", "num", "num-traits", "num_cpus", @@ -14424,9 +14461,9 @@ checksum = "81b79ad29b5e19de4260020f8919b443b2ef0277d242ce532ec7b7a2cc8b6007" [[package]] name = "unicode-ident" -version = "1.0.19" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "unicode-linebreak" @@ -14436,9 +14473,9 @@ checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" [[package]] name = "unicode-normalization" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" dependencies = [ "tinyvec", ] @@ -14454,9 +14491,9 @@ dependencies = [ [[package]] name = "unicode-properties" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-segmentation" @@ -14472,9 +14509,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] name = "unicode-xid" @@ -14630,7 +14667,7 @@ version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "js-sys", "serde", "wasm-bindgen", @@ -14738,20 +14775,11 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasi" -version = "0.14.5+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4494f6290a82f5fe584817a676a34b9d6763e8d9d18204009fb31dceca98fd4" -dependencies = [ - "wasip2", -] - [[package]] name = "wasip2" -version = "1.0.0+wasi-0.2.4" +version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03fa2761397e5bd52002cd7e73110c71af2109aca4e521a9f40473fe685b0a24" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ "wit-bindgen", ] @@ -14764,9 +14792,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.101" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e14915cadd45b529bb8d1f343c4ed0ac1de926144b746e2710f9cd05df6603b" +checksum = "ab10a69fbd0a177f5f649ad4d8d3305499c42bab9aef2f7ff592d0ec8f833819" dependencies = [ "cfg-if", "once_cell", @@ -14777,23 +14805,23 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.101" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28d1ba982ca7923fd01448d5c30c6864d0a14109560296a162f80f305fb93bb" +checksum = "0bb702423545a6007bbc368fde243ba47ca275e549c8a28617f56f6ba53b1d1c" dependencies = [ "bumpalo", "log", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.51" +version = "0.4.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca85039a9b469b38336411d6d6ced91f3fc87109a2a27b0c197663f5144dffe" +checksum = "a0b221ff421256839509adbb55998214a70d829d3a28c69b4a6672e9d2a42f67" dependencies = [ "cfg-if", "js-sys", @@ -14804,9 +14832,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.101" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3d463ae3eff775b0c45df9da45d68837702ac35af998361e2c84e7c5ec1b0d" +checksum = "fc65f4f411d91494355917b605e1480033152658d71f722a90647f56a70c88a0" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -14814,31 +14842,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.101" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bb4ce89b08211f923caf51d527662b75bdc9c9c7aab40f86dcb9fb85ac552aa" +checksum = "ffc003a991398a8ee604a401e194b6b3a39677b3173d6e74495eb51b82e99a32" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.101" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f143854a3b13752c6950862c906306adb27c7e839f7414cec8fea35beab624c1" +checksum = "293c37f4efa430ca14db3721dfbe48d8c33308096bd44d80ebaa775ab71ba1cf" dependencies = [ "unicode-ident", ] [[package]] name = "wasm-bindgen-test" -version = "0.3.51" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80cc7f8a4114fdaa0c58383caf973fc126cf004eba25c9dc639bccd3880d55ad" +checksum = "aee0a0f5343de9221a0d233b04520ed8dc2e6728dce180b1dcd9288ec9d9fa3c" dependencies = [ "js-sys", "minicov", @@ -14849,13 +14877,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.51" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5ada2ab788d46d4bda04c9d567702a79c8ced14f51f221646a16ed39d0e6a5d" +checksum = "a369369e4360c2884c3168d22bded735c43cccae97bbc147586d4b480edd138d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -14893,14 +14921,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eeee3bdea6257cc36d756fa745a70f9d393571e47d69e0ed97581676a5369ca" dependencies = [ "deno_error", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "web-sys" -version = "0.3.78" +version = "0.3.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e4b637749ff0d92b8fad63aa1f7cff3cbe125fd49c175cd6345e7272638b12" +checksum = "fbe734895e869dc429d78c4b433f8d17d95f8d05317440b4fad5ab2d33e596dc" dependencies = [ "js-sys", "wasm-bindgen", @@ -14922,14 +14950,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75c7f0ef91146ebfb530314f5f1d24528d7f0767efbfd31dce919275413e393e" dependencies = [ - "webpki-root-certs 1.0.2", + "webpki-root-certs 1.0.4", ] [[package]] name = "webpki-root-certs" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4ffd8df1c57e87c325000a3d6ef93db75279dc3a231125aac571650f22b12a" +checksum = "ee3e3b5f5e80bc89f30ce8d0343bf4e5f12341c51f3e26cbeecbc7c85443e85b" dependencies = [ "rustls-pki-types", ] @@ -14940,14 +14968,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.2", + "webpki-roots 1.0.4", ] [[package]] name = "webpki-roots" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" +checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" dependencies = [ "rustls-pki-types", ] @@ -14968,7 +14996,7 @@ dependencies = [ "log", "naga", "once_cell", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "profiling", "raw-window-handle", "ron", @@ -15003,14 +15031,14 @@ dependencies = [ "js-sys", "khronos-egl", "libc", - "libloading 0.8.8", + "libloading 0.8.9", "log", "metal", "naga", "ndk-sys", "objc", "once_cell", - "parking_lot 0.12.4", + "parking_lot 0.12.5", "profiling", "range-alloc", "raw-window-handle", @@ -15072,9 +15100,9 @@ dependencies = [ [[package]] name = "widestring" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd7cf3379ca1aac9eea11fba24fd7e315d621f8dfe35c8d7d2be8b793726e07d" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" [[package]] name = "winapi" @@ -15098,7 +15126,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.0", + "windows-sys 0.61.2", ] [[package]] @@ -15109,13 +15137,14 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "aws-sdk-config", "aws-sigv4", "axum", "base64 0.22.1", + "bitflags 2.9.4", "chrono", "constant_time_eq", "deno_core", @@ -15123,18 +15152,20 @@ dependencies = [ "futures", "gethostname", "git-version", + "globset", "k8s-openapi", "kube", "lazy_static", - "libloading 0.8.8", + "libloading 0.8.9", "memchr", + "nom 8.0.0", "object_store", "once_cell", "pep440_rs", "prometheus", "quote", "rand 0.9.0", - "reqwest 0.12.23", + "reqwest 0.12.24", "rustls 0.23.29", "serde", "serde_json", @@ -15166,7 +15197,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "argon2", @@ -15232,7 +15263,7 @@ dependencies = [ "rand 0.9.0", "rdkafka", "regex", - "reqwest 0.12.23", + "reqwest 0.12.24", "rmcp", "rsa", "rumqttc", @@ -15249,7 +15280,7 @@ dependencies = [ "sql-builder", "sqlx", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "time", "tinyvector", "tokenizers", @@ -15286,7 +15317,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.542.1" +version = "1.573.3" dependencies = [ "base64 0.22.1", "chrono", @@ -15301,9 +15332,10 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.542.1" +version = "1.573.3" dependencies = [ "chrono", + "lazy_static", "serde", "serde_json", "sql-builder", @@ -15314,7 +15346,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "axum", @@ -15323,7 +15355,7 @@ dependencies = [ "serde", "serde_json", "sqlx", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", "uuid", @@ -15333,7 +15365,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "async-recursion", @@ -15345,6 +15377,7 @@ dependencies = [ "axum", "backon", "base64 0.22.1", + "bitflags 2.9.4", "bytes", "chrono", "chrono-tz", @@ -15357,6 +15390,7 @@ dependencies = [ "futures-core", "gethostname", "git-version", + "globset", "hex", "hmac", "hyper 1.7.0", @@ -15378,10 +15412,11 @@ dependencies = [ "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.12.23", + "reqwest 0.12.24", "reqwest-middleware", "reqwest-retry", - "semver 1.0.26", + "rmcp", + "semver 1.0.27", "serde", "serde_json", "sha2 0.10.9", @@ -15393,7 +15428,7 @@ dependencies = [ "systemstat", "tar", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "tikv-jemalloc-ctl", "tokio", "tokio-stream", @@ -15415,7 +15450,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.542.1" +version = "1.573.3" dependencies = [ "regex", "serde", @@ -15430,7 +15465,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "bytes", @@ -15454,19 +15489,19 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.542.1" +version = "1.573.3" dependencies = [ "itertools 0.14.0", "lazy_static", "proc-macro2", "quote", "regex", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] name = "windmill-parser" -version = "1.542.1" +version = "1.573.3" dependencies = [ "convert_case 0.6.0", "serde", @@ -15475,7 +15510,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "lazy_static", @@ -15487,7 +15522,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "serde_json", @@ -15499,7 +15534,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "gosyn", @@ -15511,7 +15546,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "lazy_static", @@ -15523,7 +15558,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "serde_json", @@ -15535,7 +15570,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "nu-parser", @@ -15546,7 +15581,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15557,7 +15592,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15569,7 +15604,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "async-recursion", @@ -15592,7 +15627,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "lazy_static", @@ -15606,7 +15641,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15616,14 +15651,14 @@ dependencies = [ "quote", "regex", "serde_json", - "syn 2.0.106", + "syn 2.0.109", "toml", "windmill-parser", ] [[package]] name = "windmill-parser-sql" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "lazy_static", @@ -15637,7 +15672,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "lazy_static", @@ -15653,36 +15688,12 @@ dependencies = [ "windmill-parser", ] -[[package]] -name = "windmill-parser-wasm" -version = "1.542.1" -dependencies = [ - "anyhow", - "getrandom 0.2.16", - "serde_json", - "wasm-bindgen", - "wasm-bindgen-test", - "windmill-parser", - "windmill-parser-bash", - "windmill-parser-csharp", - "windmill-parser-go", - "windmill-parser-graphql", - "windmill-parser-java", - "windmill-parser-nu", - "windmill-parser-php", - "windmill-parser-py", - "windmill-parser-ruby", - "windmill-parser-rust", - "windmill-parser-sql", - "windmill-parser-ts", - "windmill-parser-yaml", -] - [[package]] name = "windmill-parser-yaml" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", + "serde", "serde_json", "windmill-parser", "yaml-rust", @@ -15690,7 +15701,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "async-recursion", @@ -15706,8 +15717,9 @@ dependencies = [ "itertools 0.14.0", "lazy_static", "prometheus", + "quick_cache", "regex", - "reqwest 0.12.23", + "reqwest 0.12.24", "serde", "serde_json", "serde_urlencoded", @@ -15723,7 +15735,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.542.1" +version = "1.573.3" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15733,12 +15745,13 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.542.1" +version = "1.573.3" dependencies = [ "anyhow", "async-once-cell", "async-recursion", "async-stream", + "async-trait", "backon", "base64 0.22.1", "bit-vec 0.6.3", @@ -15771,8 +15784,10 @@ dependencies = [ "itertools 0.14.0", "jsonwebtoken 8.3.0", "lazy_static", - "libloading 0.8.8", + "libffi-sys", + "libloading 0.8.9", "mappable-rc", + "mime_guess", "mysql_async", "native-tls", "nix 0.27.1", @@ -15787,8 +15802,9 @@ dependencies = [ "prometheus", "rand 0.9.0", "regex", - "reqwest 0.12.23", + "reqwest 0.12.24", "reqwest-middleware", + "rmcp", "rust_decimal", "serde", "serde_json", @@ -15901,11 +15917,24 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ - "windows-implement 0.60.0", - "windows-interface 0.59.1", + "windows-implement 0.60.2", + "windows-interface 0.59.3", "windows-link 0.1.3", "windows-result 0.3.4", - "windows-strings", + "windows-strings 0.4.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement 0.60.2", + "windows-interface 0.59.3", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", ] [[package]] @@ -15927,7 +15956,7 @@ checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -15938,18 +15967,18 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] name = "windows-implement" -version = "0.60.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -15960,7 +15989,7 @@ checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -15971,18 +16000,18 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] name = "windows-interface" -version = "0.59.1" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -15993,9 +16022,9 @@ checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-link" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-numerics" @@ -16015,7 +16044,7 @@ checksum = "5b8a9ed28765efc97bbc954883f4e6796c33a06546ebafacbabee9696967499e" dependencies = [ "windows-link 0.1.3", "windows-result 0.3.4", - "windows-strings", + "windows-strings 0.4.2", ] [[package]] @@ -16036,6 +16065,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-strings" version = "0.4.2" @@ -16045,6 +16083,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -16078,16 +16125,16 @@ version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.53.3", + "windows-targets 0.53.5", ] [[package]] name = "windows-sys" -version = "0.61.0" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link 0.2.0", + "windows-link 0.2.1", ] [[package]] @@ -16123,19 +16170,19 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.3" +version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "windows-link 0.1.3", - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", + "windows-link 0.2.1", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -16161,9 +16208,9 @@ checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" [[package]] name = "windows_aarch64_msvc" @@ -16179,9 +16226,9 @@ checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_aarch64_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" [[package]] name = "windows_i686_gnu" @@ -16197,9 +16244,9 @@ checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnu" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" [[package]] name = "windows_i686_gnullvm" @@ -16209,9 +16256,9 @@ checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" [[package]] name = "windows_i686_msvc" @@ -16227,9 +16274,9 @@ checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_i686_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" [[package]] name = "windows_x86_64_gnu" @@ -16245,9 +16292,9 @@ checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnu" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" [[package]] name = "windows_x86_64_gnullvm" @@ -16263,9 +16310,9 @@ checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" [[package]] name = "windows_x86_64_msvc" @@ -16281,9 +16328,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "windows_x86_64_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" @@ -16330,15 +16377,15 @@ checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" [[package]] name = "wit-bindgen" -version = "0.45.1" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c573471f125075647d03df72e026074b7203790d41351cd6edc96f46bcccd36" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "writeable" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "wtf8" @@ -16386,9 +16433,9 @@ dependencies = [ [[package]] name = "xattr" -version = "1.5.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af3a19837351dc82ba89f8a125e22a3c475f05aba604acc023d62b2739ae2909" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", "rustix 1.1.2", @@ -16396,9 +16443,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.27" +version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fd8403733700263c6eb89f192880191f1b83e332f7a20371ddcf421c4a337c7" +checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" [[package]] name = "xmlparser" @@ -16444,13 +16491,12 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" dependencies = [ - "serde", "stable_deref_trait", - "yoke-derive 0.8.0", + "yoke-derive 0.8.1", "zerofrom", ] @@ -16462,19 +16508,19 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", "synstructure 0.13.2", ] [[package]] name = "yoke-derive" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", "synstructure 0.13.2", ] @@ -16495,7 +16541,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] @@ -16515,15 +16561,15 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", "synstructure 0.13.2", ] [[package]] name = "zeroize" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" dependencies = [ "zeroize_derive", ] @@ -16536,40 +16582,40 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] name = "zerotrie" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" dependencies = [ "displaydoc", - "yoke 0.8.0", + "yoke 0.8.1", "zerofrom", ] [[package]] name = "zerovec" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" dependencies = [ - "yoke 0.8.0", + "yoke 0.8.1", "zerofrom", "zerovec-derive", ] [[package]] name = "zerovec-derive" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.109", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index a220c089d0..68e5165f25 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.542.1" +version = "1.573.3" authors.workspace = true edition.workspace = true @@ -18,7 +18,6 @@ members = [ "./windmill-macros", "./parsers/windmill-parser", "./parsers/windmill-parser-ts", - "./parsers/windmill-parser-wasm", "./parsers/windmill-parser-go", "./parsers/windmill-parser-rust", "./parsers/windmill-parser-csharp", @@ -34,7 +33,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.542.1" +version = "1.573.3" authors = ["Ruben Fiszel "] edition = "2021" @@ -68,6 +67,7 @@ jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemal tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"] sqlx = ["windmill-worker/sqlx"] deno_core = ["windmill-worker/deno_core", "windmill-api/deno_core", "dep:deno_core", "dep:v8"] +deno_core_mac = ["deno_core", "windmill-worker/libffi_mac"] kafka = ["windmill-api/kafka"] nats = ["windmill-api/nats"] otel = ["windmill-common/otel", "windmill-worker/otel"] @@ -85,6 +85,7 @@ oauth2 = ["windmill-api/oauth2"] zip = ["windmill-api/zip"] static_frontend = ["windmill-api/static_frontend"] scoped_cache = ["windmill-common/scoped_cache"] +test_job_debouncing = [] # Languages python = ["windmill-worker/python", "windmill-api/python"] rust = ["windmill-worker/rust"] @@ -101,6 +102,10 @@ ruby = ["windmill-worker/ruby"] all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java", "ruby"] # For windows we have another set of languages enabled all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"] +all_sqlx_features = ["all_languages", "enterprise", "enterprise_saml", "loki", "embedding", "parquet", "prometheus", "flow_testing", + "openidconnect", "cloud", "jemalloc", "tantivy", "sqlx", "kafka", "nats", "otel", "dind", "websocket", "http_trigger", + "postgres_trigger", "mcp", "mqtt_trigger", "sqs_trigger", "gcp_trigger", "smtp", "stripe", + "license", "oauth2", "zip", "static_frontend", "scoped_cache", "agent_worker_server"] [patch.crates-io] object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7" } @@ -151,6 +156,10 @@ aws-sdk-config.workspace = true kube.workspace = true k8s-openapi.workspace = true libloading.workspace = true +bitflags.workspace = true +nom.workspace = true +globset.workspace = true + [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { optional = true, workspace = true } @@ -197,16 +206,18 @@ windmill-api-client = { path = "./windmill-api-client" } reqwest-retry = "^0" reqwest-middleware = { version = "^0", features = ["json"] } +bitflags = "2.9.4" memchr = "2.7.4" -axum = { version = "^0.7", features = ["multipart"] } +axum = { version = "^0.7", features = ["multipart", "macros"] } headers = "^0" hyper = { version = "^1", features = ["full"] } tokio = { version = "=1.46.1", features = ["full", "tracing", "time"] } tokio-stream = { version = "0.1.17" } tower = "^0" -tower-http = { version = "^0.6", features = ["trace", "cors"] } +tower-http = { version = "^0.6", features = ["trace", "cors", "catch-panic"] } tower-cookies = "^0.10" -serde = "^1" +#stuck because of swc for now +serde = "=1.0.219" serde_json = { version = "^1", features = ["preserve_order", "raw_value"] } serde_yml = "0.0.12" uuid = { version = "^1", features = ["serde", "v4"] } @@ -254,6 +265,7 @@ aws-sigv4 = "^1.3.4" aws-sdk-config = "=1.68.0" async-trait = "0.1.88" + v8 = "=130.0.7" # Exact version NOTE: Do not forget to update version and hash in flake.nix deno_fetch = "0.214.0" deno_tls = "0.177.0" @@ -270,6 +282,9 @@ deno_runtime = { version = "0.198.0", features = ["transpile"] } deno_telemetry = "0.12.0" deno_error = "=0.5.5" +# only used with special deno_core_mac feature to prevent ffi issue on macos, requires libffi to be installed +libffi-sys = { version = "2.3.0", features = ["system"]} + google-cloud-pubsub = "0.30.0" google-cloud-googleapis = {version = "0.16.1", features = ["pubsub"]} # TODO: remove once deno fixes the issue on their end @@ -315,7 +330,7 @@ once_cell = "1.17.1" gosyn = "0.2.6" bytes = "1.4.0" gethostname = "0.4.3" -wasm-bindgen = "^0" +wasm-bindgen = "=0.2.103" serde-wasm-bindgen = "^0" wasm-bindgen-test = "^0" convert_case = "0.6.0" @@ -355,6 +370,8 @@ pg_escape = "0.1.1" async-nats = "0.38.0" nkeys = "0.4.4" nu-parser = { version = "0.101.0", default-features = false } +nom = "8.0.0" +globset = "0.4.16" process-wrap = { version = "8.2.1", features = ["tokio1"] } diff --git a/backend/all_features_oss.sh b/backend/all_features_oss.sh index 5871bc656f..8ac9a1ea2b 100755 --- a/backend/all_features_oss.sh +++ b/backend/all_features_oss.sh @@ -1,8 +1,8 @@ +#!/bin/bash + # This script outputs all features except private. Usage : # > cargo build --features $(./all_features_oss.sh) -#!/bin/bash - # Path to the Cargo.toml file CARGO_TOML_PATH="./Cargo.toml" @@ -12,6 +12,7 @@ if [[ -f "$CARGO_TOML_PATH" ]]; then sed -n '/\[features\]/,/^\[/p' | \ grep -E '^[a-zA-Z0-9_-]+' | \ grep -v 'private' | \ + grep -v 'benchmark' | \ cut -d' ' -f1 | \ paste -sd ',' - else diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 484330c017..90e4b6fb36 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -254a6b563c503fb09d2dc332de70c2173b96491c +3674871005d1dc3b92fd5acfd76009a95a43288a \ No newline at end of file diff --git a/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py b/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py index 797216c364..c619650a87 100644 --- a/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py +++ b/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 """ -Script to parse the OpenAPI YAML file and generate Rust code with MCP tools. -Searches for endpoints tagged with 'x-mcp-tool: true' and creates a const array. +Script to parse the OpenAPI YAML file and generate Rust and TypeScript code with MCP tools. +Searches for endpoints tagged with 'x-mcp-tool: true' and creates: +- Rust code: A const array in backend/windmill-api/src/mcp/tools/auto_generated_endpoints.rs +- TypeScript code: An exported array in frontend/src/lib/mcpEndpointTools.ts """ import json @@ -219,6 +221,82 @@ def find_mcp_tools(spec: Dict[str, Any]) -> List[Dict[str, Any]]: return tools +def generate_typescript_code(tools: List[Dict[str, Any]], spec: Dict[str, Any]) -> str: + """Generate TypeScript code with MCP endpoint tools.""" + if not tools: + return """// Auto-generated MCP tools from OpenAPI specification +// This file is generated by generate_mcp_tools.py - DO NOT EDIT MANUALLY + +export interface EndpointTool { + name: string; + description: string; + instructions: string; + path: string; + method: string; + pathParamsSchema?: object; + queryParamsSchema?: object; + bodySchema?: object; +} + +export const mcpEndpointTools: EndpointTool[] = []; +""" + + tool_definitions = [] + + for tool in tools: + tool_name = tool['name'] + description = tool['description'].replace('"', '\\"').replace('\n', '\\n') + instructions = tool['instructions'].replace('"', '\\"').replace('\n', '\\n') + path = tool['path'] + method = tool['method'].upper() + + # Generate separate schemas + path_params_schema, query_params_schema, body_schema = extract_separate_schemas( + tool['parameters'], tool['requestBody'], spec, tool['required_fields'] + ) + + # Convert schemas to TypeScript - use 'as const' for better type inference + path_params_ts = json.dumps(path_params_schema, indent=8) if path_params_schema else "undefined" + query_params_ts = json.dumps(query_params_schema, indent=8) if query_params_schema else "undefined" + body_schema_ts = json.dumps(body_schema, indent=8) if body_schema else "undefined" + + # Generate tool definition + tool_def = f""" {{ + name: "{tool_name}", + description: "{description}", + instructions: "{instructions}", + path: "{path}", + method: "{method}", + pathParamsSchema: {path_params_ts}, + queryParamsSchema: {query_params_ts}, + bodySchema: {body_schema_ts} + }}""" + tool_definitions.append(tool_def) + + # Combine everything + tool_definitions_str = ",\n".join(tool_definitions) + + typescript_code = f"""// Auto-generated MCP tools from OpenAPI specification +// This file is generated by generate_mcp_tools.py - DO NOT EDIT MANUALLY + +export interface EndpointTool {{ + name: string; + description: string; + instructions: string; + path: string; + method: string; + pathParamsSchema?: object; + queryParamsSchema?: object; + bodySchema?: object; +}} + +export const mcpEndpointTools: EndpointTool[] = [ +{tool_definitions_str} +]; +""" + + return typescript_code + def generate_rust_code(tools: List[Dict[str, Any]], spec: Dict[str, Any]) -> str: """Generate the complete Rust code with MCP tools.""" if not tools: @@ -230,25 +308,25 @@ pub fn all_tools() -> Vec { vec![] } """ - + tool_definitions = [] - + for tool in tools: tool_name = tool['name'] description = tool['description'] instructions = tool['instructions'] path = tool['path'] method = tool['method'].upper() - + # Generate separate schemas path_params_schema, query_params_schema, body_schema = extract_separate_schemas( tool['parameters'], tool['requestBody'], spec, tool['required_fields'] ) - + path_params_rust = schema_to_rust_value(path_params_schema) query_params_rust = schema_to_rust_value(query_params_schema) body_schema_rust = schema_to_rust_value(body_schema) - + # Generate tool definition tool_def = f""" EndpointTool {{ name: Cow::Borrowed("{tool_name}"), @@ -261,10 +339,10 @@ pub fn all_tools() -> Vec { body_schema: {body_schema_rust}, }}""" tool_definitions.append(tool_def) - + # Combine everything tool_definitions_str = ",\n".join(tool_definitions) - + rust_code = f"""// Auto-generated MCP tools from OpenAPI specification // This file is generated by generate_mcp_tools.py - DO NOT EDIT MANUALLY @@ -276,41 +354,54 @@ pub fn all_tools() -> Vec {{ ] }} """ - + return rust_code def main(): - """Main function to parse OpenAPI and generate Rust code.""" + """Main function to parse OpenAPI and generate Rust and TypeScript code.""" script_dir = Path(__file__).parent backend_dir = script_dir.parent + project_dir = backend_dir.parent + openapi_file = backend_dir / "windmill-api" / "openapi.yaml" - output_file = backend_dir / "windmill-api" / "src" / "mcp" / "tools" / "auto_generated_endpoints.rs" - + rust_output_file = backend_dir / "windmill-api" / "src" / "mcp" / "tools" / "auto_generated_endpoints.rs" + ts_output_file = project_dir / "frontend" / "src" / "lib" / "mcpEndpointTools.ts" + if not openapi_file.exists(): print(f"OpenAPI file not found: {openapi_file}", file=sys.stderr) sys.exit(1) - + print(f"Loading OpenAPI specification from: {openapi_file}") spec = load_openapi_spec(str(openapi_file)) - + print("Searching for endpoints with x-mcp-tool: true...") tools = find_mcp_tools(spec) - + if tools: print(f"Found {len(tools)} MCP tool(s):") for tool in tools: print(f" - {tool['name']}: {tool['method']} {tool['path']}") else: print("No MCP tools found (no endpoints with x-mcp-tool: true)") - + + # Generate and write Rust code print(f"Generating Rust code...") rust_code = generate_rust_code(tools, spec) - - print(f"Writing to: {output_file}") - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, 'w', encoding='utf-8') as f: + + print(f"Writing Rust code to: {rust_output_file}") + rust_output_file.parent.mkdir(parents=True, exist_ok=True) + with open(rust_output_file, 'w', encoding='utf-8') as f: f.write(rust_code) - + + # Generate and write TypeScript code + print(f"Generating TypeScript code...") + typescript_code = generate_typescript_code(tools, spec) + + print(f"Writing TypeScript code to: {ts_output_file}") + ts_output_file.parent.mkdir(parents=True, exist_ok=True) + with open(ts_output_file, 'w', encoding='utf-8') as f: + f.write(typescript_code) + print("Done!") if __name__ == "__main__": diff --git a/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.down.sql b/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.down.sql new file mode 100644 index 0000000000..144aab482d --- /dev/null +++ b/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE gcp_trigger DROP COLUMN ack_deadline; \ No newline at end of file diff --git a/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.up.sql b/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.up.sql new file mode 100644 index 0000000000..7a85bd7c82 --- /dev/null +++ b/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE gcp_trigger ADD COLUMN ack_deadline INTEGER; \ No newline at end of file diff --git a/backend/migrations/20250923094837_websocket_can_return_error_result.down.sql b/backend/migrations/20250923094837_websocket_can_return_error_result.down.sql new file mode 100644 index 0000000000..51456cbf29 --- /dev/null +++ b/backend/migrations/20250923094837_websocket_can_return_error_result.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE websocket_trigger DROP COLUMN can_return_error_result; \ No newline at end of file diff --git a/backend/migrations/20250923094837_websocket_can_return_error_result.up.sql b/backend/migrations/20250923094837_websocket_can_return_error_result.up.sql new file mode 100644 index 0000000000..085a575e38 --- /dev/null +++ b/backend/migrations/20250923094837_websocket_can_return_error_result.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE websocket_trigger ADD COLUMN can_return_error_result BOOLEAN NOT NULL DEFAULT FALSE; \ No newline at end of file diff --git a/backend/migrations/20250924150953_new_job_result_stream_table.down.sql b/backend/migrations/20250924150953_new_job_result_stream_table.down.sql new file mode 100644 index 0000000000..5d3cb6c6fe --- /dev/null +++ b/backend/migrations/20250924150953_new_job_result_stream_table.down.sql @@ -0,0 +1,4 @@ +-- Add down migration script here +DROP TABLE job_result_stream_v2; + +ALTER TABLE job_result_stream ADD CONSTRAINT fk_job_result_stream_job_id FOREIGN KEY (job_id) REFERENCES v2_job_queue(id) ON DELETE CASCADE; \ No newline at end of file diff --git a/backend/migrations/20250924150953_new_job_result_stream_table.up.sql b/backend/migrations/20250924150953_new_job_result_stream_table.up.sql new file mode 100644 index 0000000000..c5230e0645 --- /dev/null +++ b/backend/migrations/20250924150953_new_job_result_stream_table.up.sql @@ -0,0 +1,13 @@ +-- Add up migration script here +CREATE TABLE job_result_stream_v2 ( + job_id UUID NOT NULL, + workspace_id TEXT NOT NULL, + stream TEXT NOT NULL, + idx INT NOT NULL, + PRIMARY KEY (job_id, idx) +); + +GRANT ALL ON TABLE job_result_stream_v2 TO windmill_admin; +GRANT ALL ON TABLE job_result_stream_v2 TO windmill_user; + +ALTER TABLE job_result_stream DROP CONSTRAINT fk_job_result_stream_job_id; \ No newline at end of file diff --git a/backend/migrations/20250925105841_flow_conversations.down.sql b/backend/migrations/20250925105841_flow_conversations.down.sql new file mode 100644 index 0000000000..d75054a587 --- /dev/null +++ b/backend/migrations/20250925105841_flow_conversations.down.sql @@ -0,0 +1,12 @@ +-- Add down migration script here + +-- Drop indexes +DROP INDEX IF EXISTS idx_conversation_message_conversation_time; +DROP INDEX IF EXISTS idx_flow_conversation_workspace_path; + +-- Drop tables (order matters due to foreign keys) +DROP TABLE IF EXISTS flow_conversation_message; +DROP TABLE IF EXISTS flow_conversation; + +-- Drop enum +DROP TYPE IF EXISTS MESSAGE_TYPE; \ No newline at end of file diff --git a/backend/migrations/20250925105841_flow_conversations.up.sql b/backend/migrations/20250925105841_flow_conversations.up.sql new file mode 100644 index 0000000000..5ce8e5253d --- /dev/null +++ b/backend/migrations/20250925105841_flow_conversations.up.sql @@ -0,0 +1,57 @@ +-- Add up migration script here + +-- Create message_type enum +CREATE TYPE MESSAGE_TYPE AS ENUM ('user', 'assistant'); + +-- Create flow_conversation table +CREATE TABLE flow_conversation ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id), + flow_path VARCHAR(255) NOT NULL, + title VARCHAR(255), + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + created_by VARCHAR(50) NOT NULL +); + +-- Create flow_conversation_message table +CREATE TABLE flow_conversation_message ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + conversation_id UUID NOT NULL REFERENCES flow_conversation(id) ON DELETE CASCADE, + message_type MESSAGE_TYPE NOT NULL, + content TEXT NOT NULL, + job_id UUID REFERENCES v2_job(id) ON DELETE CASCADE, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); + +-- Basic indexes for performance +CREATE INDEX idx_flow_conversation_workspace_path ON flow_conversation(workspace_id, flow_path, updated_at DESC); +CREATE INDEX idx_conversation_message_conversation_time ON flow_conversation_message(conversation_id, created_at DESC); + +-- Grant permissions +GRANT ALL ON flow_conversation TO windmill_admin; +GRANT ALL ON flow_conversation TO windmill_user; +GRANT ALL ON flow_conversation_message TO windmill_admin; +GRANT ALL ON flow_conversation_message TO windmill_user; + +-- RLS policies +ALTER TABLE flow_conversation ENABLE ROW LEVEL SECURITY; +ALTER TABLE flow_conversation_message ENABLE ROW LEVEL SECURITY; + +-- Admin policies - admins can access all conversations +CREATE POLICY admin_policy ON flow_conversation FOR ALL TO windmill_admin USING (true); +CREATE POLICY admin_policy ON flow_conversation_message FOR ALL TO windmill_admin USING (true); + +-- User policies - users can only access their own conversations +CREATE POLICY see_own ON flow_conversation FOR ALL TO windmill_user +USING (flow_conversation.created_by = current_setting('session.user')); + +-- Users can see messages of conversations they own +CREATE POLICY see_own ON flow_conversation_message FOR ALL TO windmill_user +USING ( + EXISTS ( + SELECT 1 FROM flow_conversation + WHERE flow_conversation.id = flow_conversation_message.conversation_id + AND flow_conversation.created_by = current_setting('session.user') + ) +); \ No newline at end of file diff --git a/backend/migrations/20250925142554_job_debouncing.down.sql b/backend/migrations/20250925142554_job_debouncing.down.sql new file mode 100644 index 0000000000..fd1c155f43 --- /dev/null +++ b/backend/migrations/20250925142554_job_debouncing.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS debounce_key; +DROP TABLE IF EXISTS debounce_stale_data; +DROP TABLE IF EXISTS debounce_obj_latest_version; diff --git a/backend/migrations/20250925142554_job_debouncing.up.sql b/backend/migrations/20250925142554_job_debouncing.up.sql new file mode 100644 index 0000000000..0c946aaaf5 --- /dev/null +++ b/backend/migrations/20250925142554_job_debouncing.up.sql @@ -0,0 +1,19 @@ +CREATE TABLE debounce_key ( + key VARCHAR(255) NOT NULL, + job_id uuid NOT NULL, + PRIMARY KEY (key) +); + +CREATE TABLE debounce_stale_data ( + job_id uuid NOT NULL, + to_relock TEXT[], + PRIMARY KEY (job_id) +); + +-- TODO: Prune on move/deletion +-- But normally this will persist across runs. +-- CREATE TABLE unlocked_script_latest_version ( +-- key VARCHAR(255) NOT NULL, +-- version BIGINT NOT NULL, +-- PRIMARY KEY (key) +-- ); diff --git a/backend/migrations/20250930145445_add_dynamic_skip.down.sql b/backend/migrations/20250930145445_add_dynamic_skip.down.sql new file mode 100644 index 0000000000..c9eeecd8a5 --- /dev/null +++ b/backend/migrations/20250930145445_add_dynamic_skip.down.sql @@ -0,0 +1,2 @@ +-- Remove dynamic_skip column from schedule table +ALTER TABLE schedule DROP COLUMN dynamic_skip; diff --git a/backend/migrations/20250930145445_add_dynamic_skip.up.sql b/backend/migrations/20250930145445_add_dynamic_skip.up.sql new file mode 100644 index 0000000000..4879aeead9 --- /dev/null +++ b/backend/migrations/20250930145445_add_dynamic_skip.up.sql @@ -0,0 +1,4 @@ +-- Add dynamic_skip column to schedule table +-- This column stores the path to a script that validates scheduled datetimes +-- The handler receives the scheduled_for datetime and returns a boolean +ALTER TABLE schedule ADD COLUMN dynamic_skip VARCHAR(1000) DEFAULT NULL; diff --git a/backend/migrations/20251001140645_raw_app_bundles.down.sql b/backend/migrations/20251001140645_raw_app_bundles.down.sql new file mode 100644 index 0000000000..a107fc5c11 --- /dev/null +++ b/backend/migrations/20251001140645_raw_app_bundles.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +DROP TABLE app_bundles; \ No newline at end of file diff --git a/backend/migrations/20251001140645_raw_app_bundles.up.sql b/backend/migrations/20251001140645_raw_app_bundles.up.sql new file mode 100644 index 0000000000..073faa27ee --- /dev/null +++ b/backend/migrations/20251001140645_raw_app_bundles.up.sql @@ -0,0 +1,8 @@ +-- Add up migration script here +CREATE TABLE app_bundles ( + app_version_id BIGINT NOT NULL, + w_id VARCHAR(255) NOT NULL, + file_type VARCHAR(10) NOT NULL, + data BYTEA NOT NULL, + PRIMARY KEY (app_version_id, file_type) +); diff --git a/backend/migrations/20251002151319_rename_singlescriptflow_to_singlestepflow.down.sql b/backend/migrations/20251002151319_rename_singlescriptflow_to_singlestepflow.down.sql new file mode 100644 index 0000000000..45ffaabec5 --- /dev/null +++ b/backend/migrations/20251002151319_rename_singlescriptflow_to_singlestepflow.down.sql @@ -0,0 +1,2 @@ +-- Revert singlestepflow back to singlescriptflow in job_kind enum +ALTER TYPE job_kind RENAME VALUE 'singlestepflow' TO 'singlescriptflow'; diff --git a/backend/migrations/20251002151319_rename_singlescriptflow_to_singlestepflow.up.sql b/backend/migrations/20251002151319_rename_singlescriptflow_to_singlestepflow.up.sql new file mode 100644 index 0000000000..eb97db090d --- /dev/null +++ b/backend/migrations/20251002151319_rename_singlescriptflow_to_singlestepflow.up.sql @@ -0,0 +1,2 @@ +-- Rename singlescriptflow to singlestepflow in job_kind enum +ALTER TYPE job_kind RENAME VALUE 'singlescriptflow' TO 'singlestepflow'; diff --git a/backend/migrations/20251003145612_add_end_user_to_job_perms.down.sql b/backend/migrations/20251003145612_add_end_user_to_job_perms.down.sql new file mode 100644 index 0000000000..a6834882c7 --- /dev/null +++ b/backend/migrations/20251003145612_add_end_user_to_job_perms.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE job_perms DROP COLUMN end_user_email; \ No newline at end of file diff --git a/backend/migrations/20251003145612_add_end_user_to_job_perms.up.sql b/backend/migrations/20251003145612_add_end_user_to_job_perms.up.sql new file mode 100644 index 0000000000..775fea57f5 --- /dev/null +++ b/backend/migrations/20251003145612_add_end_user_to_job_perms.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE job_perms ADD COLUMN end_user_email VARCHAR(255); \ No newline at end of file diff --git a/backend/migrations/20251006143820_ducklake_safety_migration.down.sql b/backend/migrations/20251006143820_ducklake_safety_migration.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20251006143820_ducklake_safety_migration.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20251006143820_ducklake_safety_migration.up.sql b/backend/migrations/20251006143820_ducklake_safety_migration.up.sql new file mode 100644 index 0000000000..d08237c4f8 --- /dev/null +++ b/backend/migrations/20251006143820_ducklake_safety_migration.up.sql @@ -0,0 +1,37 @@ + +-- Users of instance_settings.yaml would have issues where it deletes ducklake_user_pg_pwd +-- and then the next migration fails because it tries to insert a NULL value + +-- When everything is fine (i.e ducklake_user_pg_pwd or ducklake_settings is present) +-- this should be a no-op + +DO $$ +DECLARE + new_settings_value text; + old_setting_value text; +BEGIN + SELECT value INTO new_settings_value FROM global_settings WHERE name = 'ducklake_settings'; + SELECT trim(both '"' from value::text) INTO old_setting_value FROM global_settings WHERE name = 'ducklake_user_pg_pwd'; + + IF new_settings_value IS NULL AND old_setting_value IS NULL THEN + -- Copied from 20250731132157_ducklake_instance_settings.up.sql + + INSERT INTO global_settings (name, value) + VALUES ('ducklake_user_pg_pwd', ('"' || gen_random_uuid()::text || '"')::jsonb) + ON CONFLICT DO NOTHING; + + -- Cannot simply create the user because Postgres expect a static string for the password + -- Also we cannot drop the user easily in the down migration because databases will depend on it + -- And we cannot drop databases in transactions (migrations) + + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'ducklake_user') THEN + EXECUTE format('CREATE USER ducklake_user WITH PASSWORD %L', old_setting_value); + ELSE + EXECUTE format('ALTER USER ducklake_user WITH PASSWORD %L', old_setting_value); + END IF; + END IF; +EXCEPTION + WHEN others THEN + RAISE NOTICE 'ducklake_user migration error, skipping.'; +END +$$; \ No newline at end of file diff --git a/backend/migrations/20251006143822_ducklake_instance_settings_better.down.sql b/backend/migrations/20251006143822_ducklake_instance_settings_better.down.sql new file mode 100644 index 0000000000..febcd42333 --- /dev/null +++ b/backend/migrations/20251006143822_ducklake_instance_settings_better.down.sql @@ -0,0 +1,6 @@ +INSERT INTO global_settings (name, value) VALUES ( + 'ducklake_user_pg_pwd', + (SELECT g2.value->'ducklake_user_pg_pwd' FROM global_settings g2 WHERE g2.name = 'ducklake_settings') +); + +DELETE FROM global_settings WHERE name = 'ducklake_settings'; \ No newline at end of file diff --git a/backend/migrations/20251006143822_ducklake_instance_settings_better.up.sql b/backend/migrations/20251006143822_ducklake_instance_settings_better.up.sql new file mode 100644 index 0000000000..cdc63e3cda --- /dev/null +++ b/backend/migrations/20251006143822_ducklake_instance_settings_better.up.sql @@ -0,0 +1,9 @@ +INSERT INTO global_settings (name, value) VALUES ( + 'ducklake_settings', + (SELECT json_build_object( + 'ducklake_user_pg_pwd', g2.value, + 'instance_catalog_db_status', '{}'::json + ) FROM global_settings g2 WHERE g2.name = 'ducklake_user_pg_pwd') +); + +DELETE FROM global_settings WHERE name = 'ducklake_user_pg_pwd'; \ No newline at end of file diff --git a/backend/migrations/20251007123506_update_conversation_message_types.down.sql b/backend/migrations/20251007123506_update_conversation_message_types.down.sql new file mode 100644 index 0000000000..1b6940791d --- /dev/null +++ b/backend/migrations/20251007123506_update_conversation_message_types.down.sql @@ -0,0 +1,3 @@ +-- Remove step_name and success columns +ALTER TABLE flow_conversation_message DROP COLUMN IF EXISTS step_name; +ALTER TABLE flow_conversation_message DROP COLUMN IF EXISTS success; \ No newline at end of file diff --git a/backend/migrations/20251007123506_update_conversation_message_types.up.sql b/backend/migrations/20251007123506_update_conversation_message_types.up.sql new file mode 100644 index 0000000000..a261e83df3 --- /dev/null +++ b/backend/migrations/20251007123506_update_conversation_message_types.up.sql @@ -0,0 +1,8 @@ +-- Add up migration script here + +-- Extend MESSAGE_TYPE enum to include 'tool' +ALTER TYPE MESSAGE_TYPE ADD VALUE 'tool'; + +-- Add step_name and success columns to flow_conversation_message table +ALTER TABLE flow_conversation_message ADD COLUMN step_name VARCHAR(255); +ALTER TABLE flow_conversation_message ADD COLUMN success BOOLEAN DEFAULT TRUE NOT NULL; \ No newline at end of file diff --git a/backend/migrations/20251014181922_backfill_instance_group_uuids.down.sql b/backend/migrations/20251014181922_backfill_instance_group_uuids.down.sql new file mode 100644 index 0000000000..d130324837 --- /dev/null +++ b/backend/migrations/20251014181922_backfill_instance_group_uuids.down.sql @@ -0,0 +1,4 @@ +-- Add down migration script here + +-- This migration is irreversible as we cannot safely remove UUIDs +-- that may already be in use by SCIM clients diff --git a/backend/migrations/20251014181922_backfill_instance_group_uuids.up.sql b/backend/migrations/20251014181922_backfill_instance_group_uuids.up.sql new file mode 100644 index 0000000000..d947b80fad --- /dev/null +++ b/backend/migrations/20251014181922_backfill_instance_group_uuids.up.sql @@ -0,0 +1,7 @@ +-- Add up migration script here + +-- Backfill UUIDs for instance groups that don't have one +-- This is needed for SCIM compatibility where groups must have stable UUIDs +UPDATE instance_group +SET id = gen_random_uuid()::text +WHERE id IS NULL; diff --git a/backend/migrations/20251016163921_http_trigger_request_type.down.sql b/backend/migrations/20251016163921_http_trigger_request_type.down.sql new file mode 100644 index 0000000000..a924e23147 --- /dev/null +++ b/backend/migrations/20251016163921_http_trigger_request_type.down.sql @@ -0,0 +1,19 @@ +-- Add down migration script here +-- Add back the is_async column +ALTER TABLE http_trigger ADD COLUMN is_async BOOLEAN; + +-- Migrate request_type values back to is_async +-- 'async' -> TRUE +-- 'sync' or 'sync_sse' -> FALSE +UPDATE http_trigger SET is_async = CASE + WHEN request_type = 'async'::REQUEST_TYPE THEN TRUE + ELSE FALSE +END; + +-- Make is_async NOT NULL with default +ALTER TABLE http_trigger ALTER COLUMN is_async SET NOT NULL; +ALTER TABLE http_trigger ALTER COLUMN is_async SET DEFAULT FALSE; + +-- Drop the request_type column and type +ALTER TABLE http_trigger DROP COLUMN request_type; +DROP TYPE REQUEST_TYPE; diff --git a/backend/migrations/20251016163921_http_trigger_request_type.up.sql b/backend/migrations/20251016163921_http_trigger_request_type.up.sql new file mode 100644 index 0000000000..980f39ee24 --- /dev/null +++ b/backend/migrations/20251016163921_http_trigger_request_type.up.sql @@ -0,0 +1,23 @@ +-- Add up migration script here +-- Create the request_type enum type +CREATE TYPE REQUEST_TYPE AS ENUM ('sync', 'async', 'sync_sse'); + +-- Add the new request_type column with a default value +ALTER TABLE http_trigger ADD COLUMN request_type REQUEST_TYPE; + +-- Migrate existing is_async values to request_type +-- is_async = FALSE -> 'sync' +-- is_async = TRUE -> 'async' +UPDATE http_trigger SET request_type = CASE + WHEN is_async = TRUE THEN 'async'::REQUEST_TYPE + ELSE 'sync'::REQUEST_TYPE +END; + +-- Make request_type NOT NULL now that all values are populated +ALTER TABLE http_trigger ALTER COLUMN request_type SET NOT NULL; + +-- Set default for new rows +ALTER TABLE http_trigger ALTER COLUMN request_type SET DEFAULT 'sync'::REQUEST_TYPE; + +-- Drop the old is_async column +ALTER TABLE http_trigger DROP COLUMN is_async; diff --git a/backend/migrations/20251017120625_job_debouncing_extra.down.sql b/backend/migrations/20251017120625_job_debouncing_extra.down.sql new file mode 100644 index 0000000000..b2e9df943e --- /dev/null +++ b/backend/migrations/20251017120625_job_debouncing_extra.down.sql @@ -0,0 +1,4 @@ +-- Add down migration script here +ALTER TABLE script DROP COLUMN IF EXISTS debounce_key; +ALTER TABLE script DROP COLUMN IF EXISTS debounce_delay_s; + diff --git a/backend/migrations/20251017120625_job_debouncing_extra.up.sql b/backend/migrations/20251017120625_job_debouncing_extra.up.sql new file mode 100644 index 0000000000..0d81abf65b --- /dev/null +++ b/backend/migrations/20251017120625_job_debouncing_extra.up.sql @@ -0,0 +1,6 @@ +-- Job debouncing feature: consolidate multiple job requests within a time window +-- This reduces redundant work when the same script/flow is triggered multiple times rapidly +-- debounce_key: Custom key template for grouping jobs (e.g., "$workspace/$path-$args[id]") +-- debounce_delay_s: Delay in seconds before job execution to allow consolidation window +ALTER TABLE script ADD COLUMN IF NOT EXISTS debounce_key VARCHAR(255); +ALTER TABLE script ADD COLUMN IF NOT EXISTS debounce_delay_s INTEGER; diff --git a/backend/migrations/20251017135038_give_debounce_key_grants.down.sql b/backend/migrations/20251017135038_give_debounce_key_grants.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20251017135038_give_debounce_key_grants.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20251017135038_give_debounce_key_grants.up.sql b/backend/migrations/20251017135038_give_debounce_key_grants.up.sql new file mode 100644 index 0000000000..de9708d81c --- /dev/null +++ b/backend/migrations/20251017135038_give_debounce_key_grants.up.sql @@ -0,0 +1,4 @@ +-- Add up migration script here + +GRANT SELECT, UPDATE ON debounce_key TO windmill_user; +GRANT SELECT, UPDATE ON debounce_key TO windmill_admin; diff --git a/backend/migrations/20251024141453_job_debouncing_index_by_job_id.down.sql b/backend/migrations/20251024141453_job_debouncing_index_by_job_id.down.sql new file mode 100644 index 0000000000..719fdc0390 --- /dev/null +++ b/backend/migrations/20251024141453_job_debouncing_index_by_job_id.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +DROP INDEX IF EXISTS idx_debounce_key_job_id; diff --git a/backend/migrations/20251024141453_job_debouncing_index_by_job_id.up.sql b/backend/migrations/20251024141453_job_debouncing_index_by_job_id.up.sql new file mode 100644 index 0000000000..baaa49400e --- /dev/null +++ b/backend/migrations/20251024141453_job_debouncing_index_by_job_id.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +CREATE INDEX IF NOT EXISTS idx_debounce_key_job_id ON debounce_key (job_id); diff --git a/backend/migrations/20251027091803_update_hub_sync_script.down.sql b/backend/migrations/20251027091803_update_hub_sync_script.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20251027091803_update_hub_sync_script.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20251027091803_update_hub_sync_script.up.sql b/backend/migrations/20251027091803_update_hub_sync_script.up.sql new file mode 100644 index 0000000000..7afd90b0f1 --- /dev/null +++ b/backend/migrations/20251027091803_update_hub_sync_script.up.sql @@ -0,0 +1,291 @@ +-- Add up migration script here +UPDATE script SET content = 'import * as wmill from "windmill-cli@1.566.1" + +export async function main() { + await wmill.hubPull({ workspace: "admins", token: process.env["WM_TOKEN"], baseUrl: process.env["BASE_URL"] }); +} +', language = 'bun', +lock = '{ + "dependencies": { + "windmill-cli": "1.566.1" + } +} +//bun.lock +{ + "lockfileVersion": 1, + "workspaces": { + "": { + "dependencies": { + "windmill-cli": "1.566.1", + }, + }, + }, + "packages": { + "@ayonli/jsext": ["@ayonli/jsext@1.9.0", "", { "dependencies": { "iconv-lite": "^0.6.3", "sudo-prompt": "^9.2.1", "ws": "^8.17.0", "zod": "^3.23.8" } }, "sha512-hIu6lQhoLr5e26lmt+vzopuZffaAyb623r4+8HlN/rhXgm2ywHslzk7UHiATdfDbfPjBARkB6cfXjVEi3aav6g=="], + + "@deno/shim-deno": ["@deno/shim-deno@0.18.2", "", { "dependencies": { "@deno/shim-deno-test": "^0.5.0", "which": "^4.0.0" } }, "sha512-oQ0CVmOio63wlhwQF75zA4ioolPvOwAoK0yuzcS5bDC1JUvH3y1GS8xPh8EOpcoDQRU4FTG8OQfxhpR+c6DrzA=="], + + "@deno/shim-deno-test": ["@deno/shim-deno-test@0.5.0", "", {}, "sha512-4nMhecpGlPi0cSzT67L+Tm+GOJqvuk8gqHBziqcUQOarnuIax1z96/gJHCSIz2Z0zhxE6Rzwb3IZXPtFh51j+w=="], + + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.11", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Xt1dOL13m8u0WE8iplx9Ibbm+hFAO0GsU2P34UNoDGvZYkY8ifSiy6Zuc1lYxfG7svWE2fzqCUmFp5HCn51gJg=="], + + "@esbuild/android-arm": ["@esbuild/android-arm@0.25.11", "", { "os": "android", "cpu": "arm" }, "sha512-uoa7dU+Dt3HYsethkJ1k6Z9YdcHjTrSb5NUy66ZfZaSV8hEYGD5ZHbEMXnqLFlbBflLsl89Zke7CAdDJ4JI+Gg=="], + + "@esbuild/android-arm64": ["@esbuild/android-arm64@0.25.11", "", { "os": "android", "cpu": "arm64" }, "sha512-9slpyFBc4FPPz48+f6jyiXOx/Y4v34TUeDDXJpZqAWQn/08lKGeD8aDp9TMn9jDz2CiEuHwfhRmGBvpnd/PWIQ=="], + + "@esbuild/android-x64": ["@esbuild/android-x64@0.25.11", "", { "os": "android", "cpu": "x64" }, "sha512-Sgiab4xBjPU1QoPEIqS3Xx+R2lezu0LKIEcYe6pftr56PqPygbB7+szVnzoShbx64MUupqoE0KyRlN7gezbl8g=="], + + "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.25.11", "", { "os": "darwin", "cpu": "arm64" }, "sha512-VekY0PBCukppoQrycFxUqkCojnTQhdec0vevUL/EDOCnXd9LKWqD/bHwMPzigIJXPhC59Vd1WFIL57SKs2mg4w=="], + + "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.25.11", "", { "os": "darwin", "cpu": "x64" }, "sha512-+hfp3yfBalNEpTGp9loYgbknjR695HkqtY3d3/JjSRUyPg/xd6q+mQqIb5qdywnDxRZykIHs3axEqU6l1+oWEQ=="], + + "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.25.11", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-CmKjrnayyTJF2eVuO//uSjl/K3KsMIeYeyN7FyDBjsR3lnSJHaXlVoAK8DZa7lXWChbuOk7NjAc7ygAwrnPBhA=="], + + "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.25.11", "", { "os": "freebsd", "cpu": "x64" }, "sha512-Dyq+5oscTJvMaYPvW3x3FLpi2+gSZTCE/1ffdwuM6G1ARang/mb3jvjxs0mw6n3Lsw84ocfo9CrNMqc5lTfGOw=="], + + "@esbuild/linux-arm": ["@esbuild/linux-arm@0.25.11", "", { "os": "linux", "cpu": "arm" }, "sha512-TBMv6B4kCfrGJ8cUPo7vd6NECZH/8hPpBHHlYI3qzoYFvWu2AdTvZNuU/7hsbKWqu/COU7NIK12dHAAqBLLXgw=="], + + "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.25.11", "", { "os": "linux", "cpu": "arm64" }, "sha512-Qr8AzcplUhGvdyUF08A1kHU3Vr2O88xxP0Tm8GcdVOUm25XYcMPp2YqSVHbLuXzYQMf9Bh/iKx7YPqECs6ffLA=="], + + "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.25.11", "", { "os": "linux", "cpu": "ia32" }, "sha512-TmnJg8BMGPehs5JKrCLqyWTVAvielc615jbkOirATQvWWB1NMXY77oLMzsUjRLa0+ngecEmDGqt5jiDC6bfvOw=="], + + "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.25.11", "", { "os": "linux", "cpu": "none" }, "sha512-DIGXL2+gvDaXlaq8xruNXUJdT5tF+SBbJQKbWy/0J7OhU8gOHOzKmGIlfTTl6nHaCOoipxQbuJi7O++ldrxgMw=="], + + "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.25.11", "", { "os": "linux", "cpu": "none" }, "sha512-Osx1nALUJu4pU43o9OyjSCXokFkFbyzjXb6VhGIJZQ5JZi8ylCQ9/LFagolPsHtgw6himDSyb5ETSfmp4rpiKQ=="], + + "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.25.11", "", { "os": "linux", "cpu": "ppc64" }, "sha512-nbLFgsQQEsBa8XSgSTSlrnBSrpoWh7ioFDUmwo158gIm5NNP+17IYmNWzaIzWmgCxq56vfr34xGkOcZ7jX6CPw=="], + + "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.25.11", "", { "os": "linux", "cpu": "none" }, "sha512-HfyAmqZi9uBAbgKYP1yGuI7tSREXwIb438q0nqvlpxAOs3XnZ8RsisRfmVsgV486NdjD7Mw2UrFSw51lzUk1ww=="], + + "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.25.11", "", { "os": "linux", "cpu": "s390x" }, "sha512-HjLqVgSSYnVXRisyfmzsH6mXqyvj0SA7pG5g+9W7ESgwA70AXYNpfKBqh1KbTxmQVaYxpzA/SvlB9oclGPbApw=="], + + "@esbuild/linux-x64": ["@esbuild/linux-x64@0.25.11", "", { "os": "linux", "cpu": "x64" }, "sha512-HSFAT4+WYjIhrHxKBwGmOOSpphjYkcswF449j6EjsjbinTZbp8PJtjsVK1XFJStdzXdy/jaddAep2FGY+wyFAQ=="], + + "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.25.11", "", { "os": "none", "cpu": "arm64" }, "sha512-hr9Oxj1Fa4r04dNpWr3P8QKVVsjQhqrMSUzZzf+LZcYjZNqhA3IAfPQdEh1FLVUJSiu6sgAwp3OmwBfbFgG2Xg=="], + + "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.25.11", "", { "os": "none", "cpu": "x64" }, "sha512-u7tKA+qbzBydyj0vgpu+5h5AeudxOAGncb8N6C9Kh1N4n7wU1Xw1JDApsRjpShRpXRQlJLb9wY28ELpwdPcZ7A=="], + + "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.25.11", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-Qq6YHhayieor3DxFOoYM1q0q1uMFYb7cSpLD2qzDSvK1NAvqFi8Xgivv0cFC6J+hWVw2teCYltyy9/m/14ryHg=="], + + "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.25.11", "", { "os": "openbsd", "cpu": "x64" }, "sha512-CN+7c++kkbrckTOz5hrehxWN7uIhFFlmS/hqziSFVWpAzpWrQoAG4chH+nN3Be+Kzv/uuo7zhX716x3Sn2Jduw=="], + + "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.25.11", "", { "os": "none", "cpu": "arm64" }, "sha512-rOREuNIQgaiR+9QuNkbkxubbp8MSO9rONmwP5nKncnWJ9v5jQ4JxFnLu4zDSRPf3x4u+2VN4pM4RdyIzDty/wQ=="], + + "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.25.11", "", { "os": "sunos", "cpu": "x64" }, "sha512-nq2xdYaWxyg9DcIyXkZhcYulC6pQ2FuCgem3LI92IwMgIZ69KHeY8T4Y88pcwoLIjbed8n36CyKoYRDygNSGhA=="], + + "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.25.11", "", { "os": "win32", "cpu": "arm64" }, "sha512-3XxECOWJq1qMZ3MN8srCJ/QfoLpL+VaxD/WfNRm1O3B4+AZ/BnLVgFbUV3eiRYDMXetciH16dwPbbHqwe1uU0Q=="], + + "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.25.11", "", { "os": "win32", "cpu": "ia32" }, "sha512-3ukss6gb9XZ8TlRyJlgLn17ecsK4NSQTmdIXRASVsiS2sQ6zPPZklNJT5GR5tE/MUarymmy8kCEf5xPCNCqVOA=="], + + "@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.11", "", { "os": "win32", "cpu": "x64" }, "sha512-D7Hpz6A2L4hzsRpPaCYkQnGOotdUpDzSGRIv9I+1ITdHROSFUWW95ZPZWQmGka1Fg7W3zFJowyn9WGwMJ0+KPA=="], + + "@isaacs/balanced-match": ["@isaacs/balanced-match@4.0.1", "", {}, "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ=="], + + "@isaacs/brace-expansion": ["@isaacs/brace-expansion@5.0.0", "", { "dependencies": { "@isaacs/balanced-match": "^4.0.1" } }, "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA=="], + + "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], + + "body-parser": ["body-parser@2.2.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.0", "http-errors": "^2.0.0", "iconv-lite": "^0.6.3", "on-finished": "^2.4.1", "qs": "^6.14.0", "raw-body": "^3.0.0", "type-is": "^2.0.0" } }, "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg=="], + + "bundle-name": ["bundle-name@4.1.0", "", { "dependencies": { "run-applescript": "^7.0.0" } }, "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q=="], + + "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], + + "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="], + + "call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="], + + "content-disposition": ["content-disposition@1.0.0", "", { "dependencies": { "safe-buffer": "5.2.1" } }, "sha512-Au9nRL8VNUut/XSzbQA38+M78dzP4D+eqg3gfJHMIHHYa3bg067xj1KxMUWj+VULbiZMowKngFFbKczUrNJ1mg=="], + + "content-type": ["content-type@1.0.5", "", {}, "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA=="], + + "cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="], + + "cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="], + + "core-util-is": ["core-util-is@1.0.3", "", {}, "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="], + + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + + "default-browser": ["default-browser@5.2.1", "", { "dependencies": { "bundle-name": "^4.1.0", "default-browser-id": "^5.0.0" } }, "sha512-WY/3TUME0x3KPYdRRxEJJvXRHV4PyPoUsxtZa78lwItwRQRHhd2U9xOscaT/YTf8uCXIAjeJOFBVEh/7FtD8Xg=="], + + "default-browser-id": ["default-browser-id@5.0.0", "", {}, "sha512-A6p/pu/6fyBcA1TRz/GqWYPViplrftcW2gZC9q79ngNCKAeR/X3gcEdXQHl4KNXV+3wgIJ1CPkJQ3IHM6lcsyA=="], + + "define-lazy-prop": ["define-lazy-prop@3.0.0", "", {}, "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg=="], + + "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], + + "diff": ["diff@8.0.2", "", {}, "sha512-sSuxWU5j5SR9QQji/o2qMvqRNYRDOcBTgsJ/DeCf4iSN4gW+gNMXM7wFIP+fdXZxoNiAnHUTGjCr+TSWXdRDKg=="], + + "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], + + "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], + + "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="], + + "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="], + + "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="], + + "es-main": ["es-main@1.4.0", "", {}, "sha512-/rYhbfGK/1E6L7TcoUqmrWbSnOlMoxahiZInSYKbhIZ4/dbclHtXEcrViu4Az9IzYNBT8LcXpPszfS47zbGpwA=="], + + "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="], + + "esbuild": ["esbuild@0.25.11", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.11", "@esbuild/android-arm": "0.25.11", "@esbuild/android-arm64": "0.25.11", "@esbuild/android-x64": "0.25.11", "@esbuild/darwin-arm64": "0.25.11", "@esbuild/darwin-x64": "0.25.11", "@esbuild/freebsd-arm64": "0.25.11", "@esbuild/freebsd-x64": "0.25.11", "@esbuild/linux-arm": "0.25.11", "@esbuild/linux-arm64": "0.25.11", "@esbuild/linux-ia32": "0.25.11", "@esbuild/linux-loong64": "0.25.11", "@esbuild/linux-mips64el": "0.25.11", "@esbuild/linux-ppc64": "0.25.11", "@esbuild/linux-riscv64": "0.25.11", "@esbuild/linux-s390x": "0.25.11", "@esbuild/linux-x64": "0.25.11", "@esbuild/netbsd-arm64": "0.25.11", "@esbuild/netbsd-x64": "0.25.11", "@esbuild/openbsd-arm64": "0.25.11", "@esbuild/openbsd-x64": "0.25.11", "@esbuild/openharmony-arm64": "0.25.11", "@esbuild/sunos-x64": "0.25.11", "@esbuild/win32-arm64": "0.25.11", "@esbuild/win32-ia32": "0.25.11", "@esbuild/win32-x64": "0.25.11" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-KohQwyzrKTQmhXDW1PjCv3Tyspn9n5GcY2RTDqeORIdIJY8yKIF7sTSopFmn/wpMPW4rdPXI0UE5LJLuq3bx0Q=="], + + "escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="], + + "etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="], + + "express": ["express@5.1.0", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.0", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA=="], + + "finalhandler": ["finalhandler@2.1.0", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-/t88Ty3d5JWQbWYgaOGCCYfXRwV1+be02WqYYlL6h0lEiUAMPM8o8qKGO01YIkOHzka2up08wvgYD0mDiI+q3Q=="], + + "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="], + + "fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="], + + "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], + + "get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="], + + "get-port": ["get-port@7.1.0", "", {}, "sha512-QB9NKEeDg3xxVwCCwJQ9+xycaz6pBB6iQ76wiWMl1927n0Kir6alPiP+yuiICLLU4jpMe08dXfpebuQppFA2zw=="], + + "get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="], + + "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], + + "has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="], + + "hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="], + + "http-errors": ["http-errors@2.0.0", "", { "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", "setprototypeof": "1.2.0", "statuses": "2.0.1", "toidentifier": "1.0.1" } }, "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ=="], + + "iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], + + "immediate": ["immediate@3.0.6", "", {}, "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ=="], + + "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], + + "ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], + + "is-docker": ["is-docker@3.0.0", "", { "bin": { "is-docker": "cli.js" } }, "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ=="], + + "is-inside-container": ["is-inside-container@1.0.0", "", { "dependencies": { "is-docker": "^3.0.0" }, "bin": { "is-inside-container": "cli.js" } }, "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA=="], + + "is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="], + + "is-wsl": ["is-wsl@3.1.0", "", { "dependencies": { "is-inside-container": "^1.0.0" } }, "sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw=="], + + "isarray": ["isarray@1.0.0", "", {}, "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="], + + "isexe": ["isexe@3.1.1", "", {}, "sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ=="], + + "jszip": ["jszip@3.7.1", "", { "dependencies": { "lie": "~3.3.0", "pako": "~1.0.2", "readable-stream": "~2.3.6", "set-immediate-shim": "~1.0.1" } }, "sha512-ghL0tz1XG9ZEmRMcEN2vt7xabrDdqHHeykgARpmZ0BiIctWxM47Vt63ZO2dnp4QYt/xJVLLy5Zv1l/xRdh2byg=="], + + "lie": ["lie@3.3.0", "", { "dependencies": { "immediate": "~3.0.5" } }, "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ=="], + + "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="], + + "media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="], + + "merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="], + + "mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="], + + "mime-types": ["mime-types@3.0.1", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA=="], + + "minimatch": ["minimatch@10.0.3", "", { "dependencies": { "@isaacs/brace-expansion": "^5.0.0" } }, "sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw=="], + + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="], + + "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], + + "on-finished": ["on-finished@2.4.1", "", { "dependencies": { "ee-first": "1.1.1" } }, "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg=="], + + "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="], + + "open": ["open@10.2.0", "", { "dependencies": { "default-browser": "^5.2.1", "define-lazy-prop": "^3.0.0", "is-inside-container": "^1.0.0", "wsl-utils": "^0.1.0" } }, "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA=="], + + "pako": ["pako@1.0.11", "", {}, "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="], + + "parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="], + + "path-to-regexp": ["path-to-regexp@8.3.0", "", {}, "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA=="], + + "process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="], + + "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], + + "qs": ["qs@6.14.0", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w=="], + + "range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="], + + "raw-body": ["raw-body@3.0.1", "", { "dependencies": { "bytes": "3.1.2", "http-errors": "2.0.0", "iconv-lite": "0.7.0", "unpipe": "1.0.0" } }, "sha512-9G8cA+tuMS75+6G/TzW8OtLzmBDMo8p1JRxN5AZ+LAp8uxGA8V8GZm4GQ4/N5QNQEnLmg6SS7wyuSmbKepiKqA=="], + + "readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], + + "router": ["router@2.2.0", "", { "dependencies": { "debug": "^4.4.0", "depd": "^2.0.0", "is-promise": "^4.0.0", "parseurl": "^1.3.3", "path-to-regexp": "^8.0.0" } }, "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ=="], + + "run-applescript": ["run-applescript@7.1.0", "", {}, "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q=="], + + "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], + + "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + + "send": ["send@1.2.0", "", { "dependencies": { "debug": "^4.3.5", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.0", "mime-types": "^3.0.1", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.1" } }, "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw=="], + + "serve-static": ["serve-static@2.2.0", "", { "dependencies": { "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "parseurl": "^1.3.3", "send": "^1.2.0" } }, "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ=="], + + "set-immediate-shim": ["set-immediate-shim@1.0.1", "", {}, "sha512-Li5AOqrZWCVA2n5kryzEmqai6bKSIvpz5oUJHPVj6+dsbD3X1ixtsY5tEnsaNpH3pFAHmG8eIHUrtEtohrg+UQ=="], + + "setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="], + + "side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="], + + "side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="], + + "side-channel-map": ["side-channel-map@1.0.1", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3" } }, "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA=="], + + "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="], + + "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="], + + "string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + + "sudo-prompt": ["sudo-prompt@9.2.1", "", {}, "sha512-Mu7R0g4ig9TUuGSxJavny5Rv0egCEtpZRNMrZaYS1vxkiIxGiGUwoezU3LazIQ+KE04hTrTfNPgxU5gzi7F5Pw=="], + + "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], + + "type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="], + + "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="], + + "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], + + "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], + + "which": ["which@4.0.0", "", { "dependencies": { "isexe": "^3.1.1" }, "bin": { "node-which": "bin/which.js" } }, "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg=="], + + "windmill-cli": ["windmill-cli@1.566.1", "", { "dependencies": { "@ayonli/jsext": "*", "@deno/shim-deno": "~0.18.0", "diff": "*", "es-main": "*", "esbuild": "*", "express": "*", "get-port": "7.1.0", "jszip": "3.7.1", "minimatch": "*", "open": "*", "ws": "*" }, "bin": { "wmill": "esm/src/main.js" } }, "sha512-dyhcg/fBjOw1GvXxsFI/L+UGgoKTXUBzzVIF7p7HMcNUkD302Uf2l2MwnbJeyYT3czJ8L2oz46/5w2Rq2u/Vhg=="], + + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + + "ws": ["ws@8.18.3", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg=="], + + "wsl-utils": ["wsl-utils@0.1.0", "", { "dependencies": { "is-wsl": "^3.1.0" } }, "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw=="], + + "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + + "http-errors/statuses": ["statuses@2.0.1", "", {}, "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="], + + "raw-body/iconv-lite": ["iconv-lite@0.7.0", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ=="], + + "readable-stream/safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], + + "string_decoder/safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], + } +}' +WHERE hash = -28028598712388162 AND workspace_id = 'admins'; \ No newline at end of file diff --git a/backend/migrations/20251027235539_workspace_slack_oauth.down.sql b/backend/migrations/20251027235539_workspace_slack_oauth.down.sql new file mode 100644 index 0000000000..769d7bb927 --- /dev/null +++ b/backend/migrations/20251027235539_workspace_slack_oauth.down.sql @@ -0,0 +1,4 @@ +-- Remove workspace-level Slack OAuth configuration fields +ALTER TABLE workspace_settings + DROP COLUMN IF EXISTS slack_oauth_client_secret, + DROP COLUMN IF EXISTS slack_oauth_client_id; diff --git a/backend/migrations/20251027235539_workspace_slack_oauth.up.sql b/backend/migrations/20251027235539_workspace_slack_oauth.up.sql new file mode 100644 index 0000000000..6c98b71c33 --- /dev/null +++ b/backend/migrations/20251027235539_workspace_slack_oauth.up.sql @@ -0,0 +1,4 @@ +-- Add workspace-level Slack OAuth configuration fields +ALTER TABLE workspace_settings + ADD COLUMN slack_oauth_client_id VARCHAR(255) DEFAULT NULL, + ADD COLUMN slack_oauth_client_secret VARCHAR(255) DEFAULT NULL; diff --git a/backend/migrations/20251028101650_add_grant_to_concurrency_key.down.sql b/backend/migrations/20251028101650_add_grant_to_concurrency_key.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20251028101650_add_grant_to_concurrency_key.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20251028101650_add_grant_to_concurrency_key.up.sql b/backend/migrations/20251028101650_add_grant_to_concurrency_key.up.sql new file mode 100644 index 0000000000..36d269238c --- /dev/null +++ b/backend/migrations/20251028101650_add_grant_to_concurrency_key.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +GRANT ALL ON concurrency_counter TO windmill_admin; +GRANT ALL ON concurrency_counter TO windmill_user; \ No newline at end of file diff --git a/backend/migrations/20251030161900_nullify_debounce_delay.down.sql b/backend/migrations/20251030161900_nullify_debounce_delay.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20251030161900_nullify_debounce_delay.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20251030161900_nullify_debounce_delay.up.sql b/backend/migrations/20251030161900_nullify_debounce_delay.up.sql new file mode 100644 index 0000000000..9a2db57d03 --- /dev/null +++ b/backend/migrations/20251030161900_nullify_debounce_delay.up.sql @@ -0,0 +1,4 @@ +-- Add up migration script here +UPDATE script +SET debounce_delay_s = NULL +WHERE debounce_delay_s = 0; diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index b4f45fa486..d57e700c0d 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -2,47 +2,32 @@ "github": { "auth_url": "https://github.com/login/oauth/authorize", "token_url": "https://github.com/login/oauth/access_token", - "scopes": [ - "workflow", - "repo" - ] + "scopes": ["workflow", "repo"] }, "gitlab": { "auth_url": "https://gitlab.com/oauth/authorize", "token_url": "https://gitlab.com/oauth/token", - "scopes": [ - "api" - ] + "scopes": ["api"] }, "bitbucket": { "auth_url": "https://bitbucket.org/site/oauth2/authorize", "token_url": "https://bitbucket.org/site/oauth2/access_token", - "scopes": [ - "repository" - ] + "scopes": ["repository"] }, "slack": { "auth_url": "https://slack.com/oauth/authorize", "token_url": "https://slack.com/api/oauth.access", - "scopes": [ - "chat:write:user", - "users:read", - "users:read.email" - ] + "scopes": ["chat:write:user", "users:read", "users:read.email"] }, "supabase_wizard": { "auth_url": "https://api.supabase.com/v1/oauth/authorize", "token_url": "https://api.supabase.com/v1/oauth/token", - "scopes": [ - "all" - ] + "scopes": ["all"] }, "gsheets": { "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", - "scopes": [ - "https://www.googleapis.com/auth/spreadsheets" - ], + "scopes": ["https://www.googleapis.com/auth/spreadsheets"], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -51,9 +36,7 @@ "gdrive": { "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", - "scopes": [ - "https://www.googleapis.com/auth/drive" - ], + "scopes": ["https://www.googleapis.com/auth/drive"], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -62,9 +45,7 @@ "gmail": { "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", - "scopes": [ - "https://www.googleapis.com/auth/gmail.send" - ], + "scopes": ["https://www.googleapis.com/auth/gmail.send"], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -73,9 +54,7 @@ "gcal": { "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", - "scopes": [ - "https://www.googleapis.com/auth/calendar.events" - ], + "scopes": ["https://www.googleapis.com/auth/calendar.events"], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -84,9 +63,7 @@ "gforms": { "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", - "scopes": [ - "https://www.googleapis.com/auth/forms" - ], + "scopes": ["https://www.googleapis.com/auth/forms"], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -95,9 +72,7 @@ "gcloud": { "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", - "scopes": [ - "https://www.googleapis.com/auth/cloud-platform" - ], + "scopes": ["https://www.googleapis.com/auth/cloud-platform"], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -128,19 +103,13 @@ "linkedin": { "auth_url": "https://www.linkedin.com/oauth/v2/authorization", "token_url": "https://www.linkedin.com/oauth/v2/accessToken", - "scopes": [ - "w_member_social", - "r_liteprofile", - "r_emailaddress" - ], + "scopes": ["w_member_social", "r_liteprofile", "r_emailaddress"], "req_body_auth": true }, "quickbooks": { "auth_url": "https://appcenter.intuit.com/connect/oauth2", "token_url": "https://oauth.platform.intuit.com/oauth2/v1/tokens/bearer", - "scopes": [ - "com.intuit.quickbooks.accounting" - ] + "scopes": ["com.intuit.quickbooks.accounting"] }, "visma": { "auth_url": "https://connect.visma.com/connect/authorize", @@ -153,6 +122,11 @@ "vismanet_erp_interactive_api:update" ] }, + "sage_intacct": { + "auth_url": "https://api.intacct.com/ia/api/v1/oauth2/authorize", + "token_url": "https://api.intacct.com/ia/api/v1/oauth2/token", + "scopes": ["offline_access"] + }, "spotify": { "auth_url": "https://accounts.spotify.com/authorize", "token_url": "https://accounts.spotify.com/api/token", @@ -175,10 +149,26 @@ "xero": { "auth_url": "https://login.xero.com/identity/connect/authorize", "token_url": "https://identity.xero.com/connect/token", - "scopes": [ - "offline_access", - "accounting.transactions" - ] + "scopes": ["offline_access", "accounting.transactions"] }, - "snowflake_oauth": {} + "zoho": { + "auth_url": "https://accounts.zoho.com/oauth/v2/auth", + "token_url": "https://accounts.zoho.com/oauth/v2/token", + "scopes": [ + "ZohoAssist.sessionapi.ALL" + ], + "extra_params": { + "access_type": "offline" + } + }, + "snowflake_oauth": {}, + "apify": { + "auth_url": "https://console.apify.com/authorize/oauth", + "token_url": "https://console-backend.apify.com/oauth/apps/token", + "scopes": [ + "profile", + "full_api_access" + ], + "extra_params": {} + } } diff --git a/backend/parsers/windmill-parser-bash/src/lib.rs b/backend/parsers/windmill-parser-bash/src/lib.rs index 69d96400dc..4e3d70c038 100644 --- a/backend/parsers/windmill-parser-bash/src/lib.rs +++ b/backend/parsers/windmill-parser-bash/src/lib.rs @@ -10,7 +10,7 @@ use regex_lite::Regex; use serde_json::json; use std::{collections::HashMap, str::FromStr}; -use windmill_parser::{Arg, MainArgSignature, Typ}; +use windmill_parser::{Arg, MainArgSignature, ObjectType, Typ}; pub fn parse_bash_sig(code: &str) -> anyhow::Result { let parsed = parse_bash_file(&code)?; @@ -45,10 +45,10 @@ pub fn parse_powershell_sig(code: &str) -> anyhow::Result { } lazy_static::lazy_static! { - static ref RE_BASH: Regex = Regex::new(r#"(?m)^(\w+)="\$(?:(\d+)|\{(\d+)\}|\{(\d+):-(.*)\})"(?:[\t ]*)?(?:#.*)?$"#).unwrap(); + static ref RE_BASH: Regex = Regex::new(r#"(?m)^(\w+)="\$(?:(\d+)|\{(\d+)\}|\{(\d+):-(.*)\})"(?:[\t ]*)?(?:#.*)?\r?$"#).unwrap(); pub static ref RE_POWERSHELL_PARAM: Regex = Regex::new(r#"(?m)param[\t ]*\(([^)]*)\)"#).unwrap(); - static ref RE_POWERSHELL_ARGS: Regex = Regex::new(r#"(?:\[(\w+)\])?\$(\w+)[\t ]*(?:=[\t ]*(?:(?:(?:"|')([^"\n\r\$]*)(?:"|'))|([\d.]+)))?"#).unwrap(); + static ref RE_POWERSHELL_ARGS: Regex = Regex::new(r#"(?:\[([\w\[\]]+)\])?\$(\w+)[\t ]*(?:=[\t ]*(?:(?:(?:"|')([^"\n\r\$]*)(?:"|'))|([\d.]+)))?\r?"#).unwrap(); } fn parse_bash_file(code: &str) -> anyhow::Result>> { @@ -124,6 +124,18 @@ fn split_pwsh_args(code: &str) -> Vec<&str> { splits } +fn parse_powershell_single_typ(typ: &str) -> Typ { + match typ.to_lowercase().as_str() { + "string" => Typ::Str(None), + "int" | "long" => Typ::Int, + "decimal" | "double" | "single" => Typ::Float, + "datetime" => Typ::Datetime, + "bool" => Typ::Bool, + "pscustomobject" => Typ::Object(ObjectType::new(None, None)), + _ => Typ::Str(None), + } +} + fn parse_powershell_file(code: &str) -> anyhow::Result>> { let param_wrapper = RE_POWERSHELL_PARAM.captures(code); let mut args = vec![]; @@ -136,12 +148,12 @@ fn parse_powershell_file(code: &str) -> anyhow::Result>> { let name = cap.get(2).unwrap().as_str().to_string(); let mut parsed_typ = if let Some(typ) = typ { - match typ.as_str() { - "string" => Some(Typ::Str(None)), - "int" | "long" => Some(Typ::Int), - "decimal" | "double" | "single" => Some(Typ::Float), - "datetime" | "DateTime" => Some(Typ::Datetime), - _ => None, + if typ.as_str().ends_with("[]") { + Some(Typ::List(Box::new(parse_powershell_single_typ( + typ.as_str().strip_suffix("[]").unwrap(), + )))) + } else { + Some(parse_powershell_single_typ(typ.as_str())) } } else { None @@ -254,7 +266,7 @@ non_required="${5:-}" #[test] fn test_parse_powershell_sig() -> anyhow::Result<()> { - let code = r#"param($Msg, [string]$Msg2, $Dflt = "default value, with comma", [int]$Nb = 3 , $Nb2 = 5.0, $Nb3 = 5, $Wahoo = $env:WAHOO)"#; + let code = r#"param($Msg, [string]$Msg2, $Dflt = "default value, with comma", [int]$Nb = 3 , $Nb2 = 5.0, $Nb3 = 5, $Wahoo = $env:WAHOO, [PSCustomObject]$Obj, [string[]]$Arr)"#; assert_eq!( parse_powershell_sig(code)?, MainArgSignature { @@ -316,6 +328,22 @@ non_required="${5:-}" default: None, has_default: false, oidx: None + }, + Arg { + otyp: None, + name: "Obj".to_string(), + typ: Typ::Object(ObjectType::new(None, None)), + default: None, + has_default: false, + oidx: None + }, + Arg { + otyp: None, + name: "Arr".to_string(), + typ: Typ::List(Box::new(Typ::Str(None))), + default: None, + has_default: false, + oidx: None } ], no_main_func: None, @@ -324,4 +352,63 @@ non_required="${5:-}" ); Ok(()) } + + #[test] + fn test_parse_bash_sig_with_crlf() -> anyhow::Result<()> { + // Test with CRLF line endings (Windows-style) + let code = "\r\ntoken=\"$1\"\r\nimage=\"$2\"\r\ndigest=\"${3:-latest with spaces}\"\r\ntext=\"$4\" # with comment\r\nnon_required=\"${5:-}\"\r\n\r\n\r\n"; + assert_eq!( + parse_bash_sig(code)?, + MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![ + Arg { + otyp: None, + name: "token".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None + }, + Arg { + otyp: None, + name: "image".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None + }, + Arg { + otyp: None, + name: "digest".to_string(), + typ: Typ::Str(None), + default: Some(json!("latest with spaces")), + has_default: true, + oidx: None + }, + Arg { + otyp: None, + name: "text".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None + }, + Arg { + otyp: None, + name: "non_required".to_string(), + typ: Typ::Str(None), + default: Some(json!("")), + has_default: true, + oidx: None + } + ], + no_main_func: None, + has_preprocessor: None + } + ); + + Ok(()) + } } diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 3b181f1078..b024ba6244 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -170,10 +170,10 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result> // This is needed because we've split off the real main function above let code_with_fake_main = format!("{}\n\ndef main(): pass", code); + let ast = Suite::parse(&code_with_fake_main, "main.py").map_err(|e| { error::Error::ExecutionErr(format!("Error parsing code for imports: {}", e.to_string())) })?; - // Note: We're still using the original code for finding pins, // as the TextRange values from the parsed AST would be based on code_with_fake_main // but we want to match against the original code @@ -363,11 +363,11 @@ async fn parse_python_imports_inner( // This way we make sure there is no multiple annotations for same script // and we get detailed span on conflicting versions - #[derive(serde::Serialize, serde::Deserialize)] - struct InlineMetadata { - requires_python: String, - dependencies: Vec, - } + // #[derive(serde::Serialize, serde::Deserialize)] + // struct InlineMetadata { + // requires_python: String, + // dependencies: Vec, + // } let find_requirements = code.lines().find_position(|x| { x.starts_with("#requirements:") diff --git a/backend/parsers/windmill-parser-rust/src/lib.rs b/backend/parsers/windmill-parser-rust/src/lib.rs index 5d4d626f4c..3999fb044f 100644 --- a/backend/parsers/windmill-parser-rust/src/lib.rs +++ b/backend/parsers/windmill-parser-rust/src/lib.rs @@ -2,7 +2,7 @@ use anyhow::anyhow; use itertools::Itertools; use quote::ToTokens; use regex::Regex; -use windmill_parser::{Arg, MainArgSignature, Typ}; +use windmill_parser::{to_snake_case, Arg, MainArgSignature, Typ}; pub fn otyp_to_string(otyp: Option) -> String { otyp.unwrap() @@ -108,7 +108,7 @@ fn parse_pat_type(p: Box) -> Typ { Typ::Unknown } } - _ => Typ::Unknown, + s => Typ::Resource(to_snake_case(s)), } } else { Typ::Unknown @@ -426,23 +426,28 @@ fn main( ret.args[3].otyp, Some("Vec < Result < MyStruct , anyhow :: Error > >".to_string()) ); - assert_eq!(ret.args[3].typ, Typ::List(Box::new(Typ::Unknown))); + assert_eq!( + ret.args[3].typ, + Typ::List(Box::new(Typ::Resource("result".into()))) + ); let code = r#" // commenting comments +struct CRes(()); fn main( my_str_slice: &str, my_String: String, mut my_mut_ref_to_string: &mut String, my_string_vec: Vec, + my_resource: CRes, ) -> Result { println!("My int is {}", my_int); }"#; let ret = parse_rust_signature(code).unwrap(); - assert_eq!(ret.args.len(), 4); + assert_eq!(ret.args.len(), 5); assert_eq!(ret.args[0].name, "my_str_slice"); assert_eq!(ret.args[0].otyp, Some("& str".to_string())); @@ -459,6 +464,10 @@ fn main( assert_eq!(ret.args[3].name, "my_string_vec"); assert_eq!(ret.args[3].otyp, Some("Vec < String >".to_string())); assert_eq!(ret.args[3].typ, Typ::List(Box::new(Typ::Str(None)))); + + assert_eq!(ret.args[4].name, "my_resource"); + assert_eq!(ret.args[4].otyp, Some("CRes".to_string())); + assert_eq!(ret.args[4].typ, Typ::Resource("c_res".to_owned())); } #[test] diff --git a/backend/parsers/windmill-parser-sql/Cargo.toml b/backend/parsers/windmill-parser-sql/Cargo.toml index e66216404e..8c853df145 100644 --- a/backend/parsers/windmill-parser-sql/Cargo.toml +++ b/backend/parsers/windmill-parser-sql/Cargo.toml @@ -20,4 +20,4 @@ anyhow.workspace = true lazy_static.workspace = true serde_json.workspace = true serde.workspace = true -nom = "8.0.0" +nom.workspace = true diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index 5856fbc4fd..0fbac5b6d0 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -2,10 +2,11 @@ use anyhow::anyhow; +use lazy_static::lazy_static; #[cfg(not(target_arch = "wasm32"))] -use regex::Regex; +use regex::{Match, Regex}; #[cfg(target_arch = "wasm32")] -use regex_lite::Regex; +use regex_lite::{Match, Regex}; use serde_json::json; @@ -491,13 +492,15 @@ fn parse_pg_file(code: &str) -> anyhow::Result>> { let mut args = vec![]; let mut hm: HashMap = HashMap::new(); for cap in RE_CODE_PGSQL.captures_iter(code) { + let typ = cap + .get(2) + .map(|cap| transform_types_with_spaces(&cap, &code)) + .unwrap_or("text"); hm.insert( cap.get(1) .and_then(|x| x.as_str().parse::().ok()) .ok_or_else(|| anyhow!("Impossible to parse arg digit"))?, - cap.get(2) - .map(|x| x.as_str().to_string()) - .unwrap_or_else(|| "text".to_string()), + typ.to_string(), ); } for (i, v) in hm.iter() { @@ -543,6 +546,37 @@ fn parse_pg_file(code: &str) -> anyhow::Result>> { Ok(Some(args)) } +// The regex doesn't parse types with space such as "character varying" +// So we look for them manually and replace them with their shorter counterpart +fn transform_types_with_spaces<'a>(cap: &Match<'a>, code: &str) -> &'a str { + lazy_static! { + static ref TYPES: [(&'static str, &'static str); 6] = [ + ("character varying", "varchar"), + ("double precision", "double"), + ("time with time zone", "timetz"), + ("time without time zone", "time"), + ("timestamp with time zone", "timestamptz"), + ("timestamp without time zone", "timestamp"), + ]; + } + let typ = &code[cap.start()..]; + for (long_type, alias) in TYPES.iter() { + let mut typ = typ; + let mut found_mismatch = false; + for token in long_type.split(' ') { + if typ.len() < token.len() || !typ[..token.len()].eq_ignore_ascii_case(token) { + found_mismatch = true; + break; + } + typ = typ[token.len()..].trim_start(); + } + if !found_mismatch { + return alias; + } + } + cap.as_str() +} + pub fn parse_sql_statement_named_params(code: &str, prefix: char) -> HashSet { let mut arg_names = HashSet::new(); run_on_sql_statement_matches( diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index 399e13aa0e..04dd345b2f 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -255,6 +255,28 @@ pub fn parse_deno_signature( let mut symbol_table: HashMap = HashMap::new(); for item in ast { + // Check for named exports (e.g., export { preprocessor } from "./other") + if let ModuleItem::ModuleDecl(ModuleDecl::ExportNamed(named_export)) = &item { + if !has_preprocessor { + for specifier in &named_export.specifiers { + if let swc_ecma_ast::ExportSpecifier::Named(spec) = specifier { + let export_name = match &spec.exported { + Some(swc_ecma_ast::ModuleExportName::Ident(ident)) => ident.sym.as_ref(), + Some(swc_ecma_ast::ModuleExportName::Str(s)) => s.value.as_ref(), + None => match &spec.orig { + swc_ecma_ast::ModuleExportName::Ident(ident) => ident.sym.as_ref(), + swc_ecma_ast::ModuleExportName::Str(s) => s.value.as_ref(), + }, + }; + if export_name == "preprocessor" { + has_preprocessor = true; + break; + } + } + } + } + } + if let ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl { decl, .. })) | ModuleItem::Stmt(Stmt::Decl(decl)) = item { diff --git a/backend/parsers/windmill-parser-ts/tests/tests.rs b/backend/parsers/windmill-parser-ts/tests/tests.rs index f5a3404df8..1d9fbda6ef 100644 --- a/backend/parsers/windmill-parser-ts/tests/tests.rs +++ b/backend/parsers/windmill-parser-ts/tests/tests.rs @@ -685,4 +685,76 @@ mod tests { } ); } + + #[test] + fn test_parse_with_preprocessor_reexport() { + // Test case for issue #6894: preprocessor re-export should be detected + let code = r#" + export { preprocessor } from "./extract_user_info_from_jwt_token"; + + export async function main(param: string) { + return param; + } + "#; + let sig = parse_deno_signature(code, false, false, None).unwrap(); + assert_eq!( + sig, + MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![Arg { + name: "param".to_string(), + otyp: None, + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None, + }], + no_main_func: Some(false), + has_preprocessor: Some(true), + } + ); + } + + #[test] + fn test_parse_with_preprocessor_reexport_renamed() { + // Test case for renamed re-exports + let code = r#" + export { preprocessor as preprocessor } from "./other"; + + export async function main(param: string) { + return param; + } + "#; + let sig = parse_deno_signature(code, false, false, None).unwrap(); + assert_eq!(sig.has_preprocessor, Some(true)); + } + + #[test] + fn test_parse_with_preprocessor_among_other_exports() { + // Test case where preprocessor is one of many exports + let code = r#" + export { foo, preprocessor, bar } from "./utils"; + + export async function main(param: string) { + return param; + } + "#; + let sig = parse_deno_signature(code, false, false, None).unwrap(); + assert_eq!(sig.has_preprocessor, Some(true)); + } + + #[test] + fn test_parse_without_preprocessor_other_exports() { + // Test case where there are exports but no preprocessor + let code = r#" + export { foo, bar } from "./utils"; + + export async function main(param: string) { + return param; + } + "#; + let sig = parse_deno_signature(code, false, false, None).unwrap(); + assert_eq!(sig.has_preprocessor, Some(false)); + } } diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 0d60735a3a..40f53ecb48 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -1,10 +1,11 @@ +cargo-features = ["panic-immediate-abort"] + [package] name = "windmill-parser-wasm" version.workspace = true edition.workspace = true authors.workspace = true - [lib] crate-type = ["cdylib"] name = "windmill_parser_wasm" @@ -15,6 +16,13 @@ wasm-bindgen-test.workspace = true windmill-parser-ts.workspace = true windmill-parser-bash.workspace = true +[profile.release] +panic = "immediate-abort" + +[unstable] +build-std = ["std", "panic_abort"] + + [features] default = [] go-parser = [ "dep:windmill-parser-go"] @@ -48,5 +56,6 @@ windmill-parser-nu = { workspace = true, optional = true } windmill-parser-java = { workspace = true, optional = true } windmill-parser-ruby = { workspace = true, optional = true } wasm-bindgen.workspace = true + serde_json.workspace = true getrandom = { workspace = true, features = ["js"] } diff --git a/backend/parsers/windmill-parser-wasm/build.nu b/backend/parsers/windmill-parser-wasm/build.nu index 1ab87f9873..7c3bf9d4a6 100755 --- a/backend/parsers/windmill-parser-wasm/build.nu +++ b/backend/parsers/windmill-parser-wasm/build.nu @@ -112,7 +112,7 @@ def main [ print $"Building in ($profile) mode ($env.OUT_DIR)" match $t.env { "default" => { - wasm-pack build ($profile) --target ($tar) --out-dir $env.OUT_DIR --features ($t.features) -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort + wasm-pack build --weak-refs ($profile) --target ($tar) --out-dir $env.OUT_DIR --features ($t.features) -Z build-std=panic_abort,std }, "tree-sitter" => { $env.CFLAGS_wasm32_unknown_unknown = $"-I(pwd)/wasm-sysroot -Wbad-function-cast -Wcast-function-type -fno-builtin" diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index 238b207dd2..f3b7929d38 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -1,7 +1,7 @@ #[cfg(feature = "ts-parser")] use serde_json::json; #[allow(unused_imports)] -use wasm_bindgen::prelude::*; +use wasm_bindgen::prelude::wasm_bindgen; use windmill_parser::MainArgSignature; #[cfg(feature = "ts-parser")] use windmill_parser_ts::{parse_expr_for_ids, parse_expr_for_imports}; @@ -150,6 +150,16 @@ pub fn parse_ansible(code: &str) -> String { wrap_sig(windmill_parser_yaml::parse_ansible_sig(code)) } +#[cfg(feature = "ansible-parser")] +#[wasm_bindgen] +pub fn parse_ansible_delegate(code: &str) -> String { + if let Ok(r) = windmill_parser_yaml::parse_delegate_to_git_repo(code) { + return serde_json::to_string(&r).unwrap(); + } else { + return "Invalid".to_string(); + } +} + #[cfg(feature = "csharp-parser")] #[wasm_bindgen] pub fn parse_csharp(code: &str) -> String { @@ -173,6 +183,7 @@ pub fn parse_java(code: &str) -> String { pub fn parse_ruby(code: &str) -> String { wrap_sig(windmill_parser_ruby::parse_ruby_signature(code)) } + #[cfg(feature = "sql-parser")] #[wasm_bindgen] pub fn parse_assets_sql(code: &str) -> String { @@ -203,4 +214,15 @@ pub fn parse_assets_py(code: &str) -> String { } } +#[cfg(feature = "ansible-parser")] +#[wasm_bindgen] +pub fn parse_assets_ansible(code: &str) -> String { + let o = windmill_parser_yaml::parse_assets(code); + if let Ok(r) = o { + return serde_json::to_string(&r).unwrap(); + } else { + return format!("err: {:?}", o.err().unwrap()); + } +} + // for related places search: ADD_NEW_LANG diff --git a/backend/parsers/windmill-parser-yaml/Cargo.toml b/backend/parsers/windmill-parser-yaml/Cargo.toml index 9bcfbea7a7..eb01ba1936 100644 --- a/backend/parsers/windmill-parser-yaml/Cargo.toml +++ b/backend/parsers/windmill-parser-yaml/Cargo.toml @@ -13,3 +13,4 @@ yaml-rust.workspace = true windmill-parser.workspace = true anyhow.workspace = true serde_json.workspace = true +serde.workspace = true diff --git a/backend/parsers/windmill-parser-yaml/src/asset_parser.rs b/backend/parsers/windmill-parser-yaml/src/asset_parser.rs new file mode 100644 index 0000000000..0b9ba0d38a --- /dev/null +++ b/backend/parsers/windmill-parser-yaml/src/asset_parser.rs @@ -0,0 +1,41 @@ +use windmill_parser::asset_parser::{ + merge_assets, AssetKind, AssetUsageAccessType, ParseAssetsResult, + }; + +use crate::{parse_ansible_reqs, ResourceOrVariablePath}; + +pub fn parse_assets(input: &str) -> anyhow::Result>> { + let mut assets = vec![]; + if let (_, Some(ansible_reqs), _) = parse_ansible_reqs(input)? { + if let Some(delegate_to_git_repo_details) = ansible_reqs.delegate_to_git_repo { + assets.push(ParseAssetsResult { + kind: AssetKind::Resource, + path: delegate_to_git_repo_details.resource, + access_type: Some(AssetUsageAccessType::R), + }) + } + + for i in ansible_reqs.inventories { + if let Some(pinned_res) = i.pinned_resource { + assets.push(ParseAssetsResult { + kind: AssetKind::Resource, + path: pinned_res, + access_type: Some(AssetUsageAccessType::R), + }) + } + } + + for file in ansible_reqs.file_resources { + if let ResourceOrVariablePath::Resource(resource) = file.resource_path { + assets.push(ParseAssetsResult { + kind: AssetKind::Resource, + path: resource, + access_type: Some(AssetUsageAccessType::R), + }) + } + } + } + + Ok(merge_assets(assets)) +} + diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index 710d17c966..5e3af4f831 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -1,15 +1,26 @@ use std::collections::HashMap; use anyhow::anyhow; +use serde::Serialize; use serde_json::json; use windmill_parser::{Arg, MainArgSignature, ObjectProperty, ObjectType, Typ}; use yaml_rust::{Yaml, YamlEmitter, YamlLoader}; +pub mod asset_parser; +pub use asset_parser::parse_assets; + pub fn parse_ansible_sig(inner_content: &str) -> anyhow::Result { let docs = YamlLoader::load_from_str(inner_content) .map_err(|e| anyhow!("Failed to parse yaml: {}", e))?; - if docs.len() < 2 { + let mut delegating_to_git_repo = false; + if let Yaml::Hash(doc) = &docs[0] { + if let Some(v) = doc.get(&Yaml::String("delegate_to_git_repo".to_string())) { + delegating_to_git_repo = extract_delegate_to_git_repo_details(v).is_some(); + } + } + + if docs.len() < 2 && !delegating_to_git_repo { return Ok(MainArgSignature { star_args: false, star_kwargs: false, @@ -56,7 +67,21 @@ pub fn parse_ansible_sig(inner_content: &str) -> anyhow::Result { + for inv in parse_additional_inventories(value)? { + if let PreexistingAnsibleInventory::PassedInArgs(i) = inv { + args.push(Arg { + name: i.name, + otyp: None, + typ: Typ::List(Box::new(Typ::Str(i.options))), + has_default: false, + default: None, + oidx: None, + }); + } } } _ => (), @@ -211,6 +236,18 @@ pub struct AnsibleInventory { pub pinned_resource: Option, } +#[derive(Debug, Clone)] +pub enum PreexistingAnsibleInventory { + Static(String), + PassedInArgs(InventoryFilenameListDefinition), +} + +#[derive(Debug, Clone)] +pub struct InventoryFilenameListDefinition { + pub options: Option>, + pub name: String, +} + #[derive(Debug, Clone)] pub struct GitRepo { pub url: String, @@ -219,12 +256,22 @@ pub struct GitRepo { pub target_path: String, } +#[derive(Debug, Clone, Serialize)] +pub struct DelegateToGitRepoDetails { + pub resource: String, + pub playbook: Option, + pub commit: Option, + pub inventories_location: Option, + pub vars_location: Option, +} + #[derive(Debug, Clone)] pub struct AnsibleRequirements { pub python_reqs: Vec, pub roles_and_collections: Option, pub file_resources: Vec, pub inventories: Vec, + pub additional_inventories: Vec, pub vars: Vec<(String, String)>, pub resources: Vec<(String, String)>, pub options: AnsiblePlaybookOptions, @@ -232,6 +279,7 @@ pub struct AnsibleRequirements { pub vault_id: Vec, pub git_repos: Vec, pub git_ssh_identity: Vec, + pub delegate_to_git_repo: Option, } impl Default for AnsibleRequirements { @@ -241,6 +289,7 @@ impl Default for AnsibleRequirements { roles_and_collections: None, file_resources: vec![], inventories: vec![], + additional_inventories: vec![], vars: vec![], resources: vec![], options: AnsiblePlaybookOptions { @@ -254,10 +303,57 @@ impl Default for AnsibleRequirements { vault_id: vec![], git_repos: vec![], git_ssh_identity: vec![], + delegate_to_git_repo: None, } } } +fn parse_additional_inventories( + inventory_yaml: &Yaml, +) -> anyhow::Result> { + if let Yaml::Array(arr) = inventory_yaml { + let mut ret = vec![]; + let mut count = -1; + for inv in arr { + if let Yaml::String(inv_name) = inv { + ret.push(PreexistingAnsibleInventory::Static(inv_name.clone())); + } else if let Yaml::Hash(inv) = inv { + if let Some(options) = inv.get(&Yaml::String("options".to_string())) { + let options = match options { + Yaml::Null => None, + Yaml::Array(elements) => Some( + elements + .iter() + .filter_map(|s| s.as_str().map(|s| s.to_string())) + .collect(), + ), + _ => continue, + }; + + let name = inv + .get(&Yaml::String("name".to_string())) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .unwrap_or_else(|| { + count += 1; + if count == 0 { + "Additional inventories".to_string() + } else { + format!("Additional inventories ({count})") + } + }); + + ret.push(PreexistingAnsibleInventory::PassedInArgs( + InventoryFilenameListDefinition { options, name }, + )) + } + } + } + return Ok(ret); + } + return Err(anyhow!("Invalid inventory definition")); +} + fn parse_inventories(inventory_yaml: &Yaml) -> anyhow::Result> { if let Yaml::Array(arr) = inventory_yaml { let mut ret = vec![]; @@ -303,6 +399,33 @@ fn parse_inventories(inventory_yaml: &Yaml) -> anyhow::Result, + git_ssh_identity: Vec, +} + +pub fn parse_delegate_to_git_repo(inner_content: &str) -> anyhow::Result { + let docs = YamlLoader::load_from_str(inner_content) + .map_err(|e| anyhow!("Failed to parse yaml: {}", e))?; + + let mut git_ssh_identity: Vec = vec![]; + + if let Yaml::Hash(doc) = &docs[0] { + if let Some(v) = doc.get(&Yaml::String("git_ssh_identity".to_string())) { + let _ = extract_ssh_identity(&v, &mut git_ssh_identity); + } + if let Some(v) = doc.get(&Yaml::String("delegate_to_git_repo".to_string())) { + return Ok(DelegateWithSSHAuth { + delegate_to_git_repo_details: extract_delegate_to_git_repo_details(v), + git_ssh_identity, + }); + } + } + + Ok(DelegateWithSSHAuth { delegate_to_git_repo_details: None, git_ssh_identity }) +} + pub fn parse_ansible_reqs( inner_content: &str, ) -> anyhow::Result<(String, Option, String)> { @@ -310,11 +433,17 @@ pub fn parse_ansible_reqs( let docs = YamlLoader::load_from_str(inner_content) .map_err(|e| anyhow!("Failed to parse yaml: {}", e))?; - if docs.len() < 2 { - return Ok((logs, None, inner_content.to_string())); + let mut ret = AnsibleRequirements::default(); + + if let Yaml::Hash(doc) = &docs[0] { + if let Some(v) = doc.get(&Yaml::String("delegate_to_git_repo".to_string())) { + ret.delegate_to_git_repo = extract_delegate_to_git_repo_details(v); + } } - let mut ret = AnsibleRequirements::default(); + if ret.delegate_to_git_repo.is_none() && docs.len() < 2 { + return Ok((logs, None, inner_content.to_string())); + } if let Yaml::Hash(doc) = &docs[0] { for (key, value) in doc { @@ -367,7 +496,11 @@ pub fn parse_ansible_reqs( } } Yaml::String(key) if key == "inventory" => { - ret.inventories = parse_inventories(value)?; + ret.inventories.extend(parse_inventories(value)?); + } + Yaml::String(key) if key == "additional_inventories" => { + ret.additional_inventories + .extend(parse_additional_inventories(value)?); } Yaml::String(key) if key == "vault_password" => { let Yaml::String(filename) = value else { @@ -407,27 +540,15 @@ pub fn parse_ansible_reqs( } } Yaml::String(key) if key == "git_ssh_identity" => { - let Yaml::Array(indentities) = &value else { - return Err(anyhow!( - "git_ssh_identity expects an array of windmill variables (or secrets) containing ssh IDs" - )); - }; - - for r in indentities { - let Yaml::String(file_name) = r else { - return Err(anyhow!( - "Git ssh identity file must be a string path to a Windmill variable/secret" - )); - }; - - ret.git_ssh_identity.push(file_name.clone()); - } + extract_ssh_identity(&value, &mut ret.git_ssh_identity)?; } + Yaml::String(key) if key == "delegate_to_git_repo" => {} // Skip this because it was already parsed before Yaml::String(key) => logs.push_str(&format!("\nUnknown field `{}`. Ignoring", key)), _ => (), } } } + let mut out_str = String::new(); let mut emitter = YamlEmitter::new(&mut out_str); @@ -437,6 +558,61 @@ pub fn parse_ansible_reqs( Ok((logs, Some(ret), out_str)) } +fn extract_ssh_identity(value: &Yaml, ret: &mut Vec) -> anyhow::Result<()> { + let Yaml::Array(indentities) = value else { + return Err(anyhow!( + "git_ssh_identity expects an array of windmill variables (or secrets) containing ssh IDs" + )); + }; + + for r in indentities { + let Yaml::String(file_name) = r else { + return Err(anyhow!( + "Git ssh identity file must be a string path to a Windmill variable/secret" + )); + }; + + ret.push(file_name.clone()); + } + Ok(()) +} + +fn extract_delegate_to_git_repo_details(value: &Yaml) -> Option { + if let Yaml::Hash(v) = value { + if let Some(resource) = v + .get(&Yaml::String("resource".to_string())) + .and_then(|s| s.as_str()) + .map(|s| s.to_string()) + { + let playbook = v + .get(&Yaml::String("playbook".to_string())) + .and_then(|s| s.as_str()) + .map(|s| s.to_string()); + let commit = v + .get(&Yaml::String("commit".to_string())) + .and_then(|s| s.as_str()) + .map(|s| s.to_string()); + let inventories_location = v + .get(&Yaml::String("inventories_location".to_string())) + .and_then(|s| s.as_str()) + .map(|s| s.to_string()); + let vars_location = v + .get(&Yaml::String("vars_location".to_string())) + .and_then(|s| s.as_str()) + .map(|s| s.to_string()); + + return Some(DelegateToGitRepoDetails { + resource, + playbook, + commit, + inventories_location, + vars_location, + }); + } + } + return None; +} + fn parse_git_repo(r: &Yaml) -> anyhow::Result { let Yaml::Hash(repo) = r else { return Err(anyhow!("Should be a Map")); @@ -679,7 +855,7 @@ pub fn add_versions_to_requirements_yaml( input: &str, role_versions: &HashMap, collection_versions: &HashMap, -) -> anyhow::Result<(String,String)> { +) -> anyhow::Result<(String, String)> { let mut docs = YamlLoader::load_from_str(input).map_err(|e| anyhow!("YAML parse error: {}", e))?; let doc = &mut docs[0]; @@ -709,3 +885,89 @@ pub fn add_versions_to_requirements_yaml( Ok((out_str, logs)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_ansible_assets() { + let p = r#" +--- +inventory: + - resource_type: ansible_inventory + # You can pin an inventory to this script by hardcoding the resource path: + # resource: u/user/your_resource +# - name: hcloud.yml +# resource_type: dynamic_inventory + +additional_inventories: + - options: ["a", "b", "c"] + +options: + - verbosity: vvv + +delegate_to_git_repo: + resource: u/admin/git_reportino + playbook: ./playbooks/playbook.yml + commit: 7sh7dh73h7dhd299d91hd1hdh3d3hygh4372 + + +# File resources will be written in the relative `target` location before +# running the playbook +files: + - resource: u/user/fabulous_jinja_template + target: ./config_template.j2 + - variable: u/user/ssh_key + target: ./ssh_key + mode: '0600' + +# Define the arguments of the windmill script +extra_vars: + world_qualifier: + type: string + +# If using Ansible Vault: +# vault_password: u/user/ansible_vault_password + +dependencies: + galaxy: + collections: + - name: community.general + - name: community.vmware + roles: + python: + - jmespath +--- +- name: Echo + hosts: 127.0.0.1 + connection: local + vars: + my_result: + a: 2 + b: true + c: "Hello" + + tasks: + - name: Print debug message + debug: + msg: "Hello, {{world_qualifier}} world!" + - name: Write variable my_result to result.json + delegate_to: localhost + copy: + content: "{{ my_result | to_json }}" + dest: result.json +"#; + let a = parse_assets(p).unwrap(); + println!("The resulting assets are: {}", a.len()); + + let a = parse_ansible_reqs(p).unwrap(); + println!("The resulting reqs are: {:#?}", a); + + let a = parse_ansible_sig(p).unwrap(); + println!("The resulting sig is: {:#?}", a); + + let a = parse_delegate_to_git_repo(p).unwrap(); + println!("The resulting delegate_to_kit_repo is: {:#?}", a); + } +} diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 82cf8b8499..a721590ae9 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -26,6 +26,13 @@ pub struct ParseAssetsResult> { pub access_type: Option, // None in case of ambiguity } +#[derive(Debug, Clone, Serialize)] +pub struct DelegateToGitRepoDetails { + pub resource: String, + pub playbook: Option, + pub commit: Option, +} + pub fn merge_assets>(assets: Vec>) -> Vec> { let mut arr: Vec> = vec![]; for asset in assets { diff --git a/backend/rust-best-practices.mdc b/backend/rust-best-practices.mdc index bdbdb0d24d..2df01d98bf 100644 --- a/backend/rust-best-practices.mdc +++ b/backend/rust-best-practices.mdc @@ -65,6 +65,7 @@ When generating code, especially involving `serde`, `sqlx`, and `tokio`, priorit ### SQLx Optimizations (Database Interaction) +- **CRITICAL - Never Use `SELECT *` in Worker-Executed Queries:** For any query that can potentially be executed by workers, **always** explicitly list the specific columns you need instead of using `SELECT *`. This is essential for backwards compatibility: when workers are running behind the API server version (common in distributed deployments), adding new columns to database tables will cause outdated workers to fail when they try to deserialize rows with unexpected columns. Always use explicit column lists like `SELECT id, workspace_id, path, created_at FROM table` instead of `SELECT * FROM table`. - **Select Only Necessary Columns:** In `SELECT` queries, list specific column names rather than using `SELECT *`. This reduces data transferred from the database and the work needed for hydration/deserialization. - **Batch Operations:** For multiple `INSERT`, `UPDATE`, or `DELETE` statements, prefer executing them in a single query if the database and driver support it efficiently (e.g., `INSERT INTO ... VALUES (...), (...), ...`). This minimizes round trips to the database. - **Avoid N+1 Queries:** Do not loop through results of one query and execute a separate query for each item (e.g., fetching users, then querying for each user's profile in a loop). Instead, use JOINs or a single query with an `IN` clause to fetch related data efficiently. diff --git a/backend/src/main.rs b/backend/src/main.rs index 78c2a1e637..70f428d4ca 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -10,9 +10,10 @@ use monitor::{ load_base_url, load_otel, reload_critical_alerts_on_db_oversize, reload_delete_logs_periodically_setting, reload_indexer_config, reload_instance_python_version_setting, reload_maven_repos_setting, - reload_no_default_maven_setting, reload_nuget_config_setting, reload_ruby_repos_setting, - reload_timeout_wait_result_setting, send_current_log_file_to_object_store, - send_logs_to_object_store, WORKERS_NAMES, + reload_no_default_maven_setting, reload_nuget_config_setting, + reload_powershell_repo_pat_setting, reload_powershell_repo_url_setting, + reload_ruby_repos_setting, reload_timeout_wait_result_setting, + send_current_log_file_to_object_store, send_logs_to_object_store, WORKERS_NAMES, }; use rand::Rng; use sqlx::postgres::PgListener; @@ -41,14 +42,15 @@ use windmill_common::{ CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_PYTHON_VERSION_SETTING, - JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, - LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, - NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, - OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, - RUBY_REPOS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING, - TIMEOUT_WAIT_RESULT_SETTING, + HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, + INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, + KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, + MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, + NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, + PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, + REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, + RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, + SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, scripts::ScriptLang, stats_oss::schedule_stats, @@ -93,10 +95,10 @@ use crate::monitor::{ reload_app_workspaced_route_setting, reload_base_url_setting, reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting, reload_critical_error_channels_setting, reload_extra_pip_index_url_setting, - reload_hub_base_url_setting, reload_job_default_timeout_setting, reload_jwt_secret_setting, - reload_license_key, reload_npm_config_registry_setting, reload_pip_index_url_setting, - reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config, - reload_worker_config, MonitorIteration, + reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_job_default_timeout_setting, + reload_jwt_secret_setting, reload_license_key, reload_npm_config_registry_setting, + reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting, + reload_smtp_config, reload_worker_config, MonitorIteration, }; #[cfg(feature = "parquet")] @@ -300,6 +302,19 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { } async fn windmill_main() -> anyhow::Result<()> { + let (killpill_tx, mut killpill_rx) = KillpillSender::new(2); + let mut monitor_killpill_rx = killpill_tx.subscribe(); + let (killpill_phase2_tx, _killpill_phase2_rx) = tokio::sync::broadcast::channel::<()>(2); + let server_killpill_rx = killpill_phase2_tx.subscribe(); + + let shutdown_tx = killpill_tx.clone(); + let shutdown_rx = killpill_tx.subscribe(); + tokio::spawn(async move { + if let Err(e) = windmill_common::shutdown_signal(shutdown_tx, shutdown_rx).await { + tracing::error!("Error in shutdown signal: {e:#}"); + } + }); + dotenv::dotenv().ok(); update_ca_certificates_if_requested(); @@ -408,7 +423,7 @@ async fn windmill_main() -> anyhow::Result<()> { ); let suffix = create_default_worker_suffix(&hostname); ( - Connection::Http(build_agent_http_client(&suffix)), + Connection::Http(build_agent_http_client(&suffix, None, None)), Some(suffix), ) } else { @@ -435,17 +450,16 @@ async fn windmill_main() -> anyhow::Result<()> { environment } else { load_base_url(&conn) - .await - .unwrap_or_else(|_| "local".to_string()) - .trim_start_matches("https://") - .trim_start_matches("http://") - .split(".") - .next() - .unwrap_or_else(|| "local") - .to_string() + .await + .unwrap_or_else(|_| "local".to_string()) + .trim_start_matches("https://") + .trim_start_matches("http://") + .split(".") + .next() + .unwrap_or_else(|| "local") + .to_string() }; - let _guard = windmill_common::tracing_init::initialize_tracing(&hostname, &mode, &environment); let is_agent = mode == Mode::Agent; @@ -464,13 +478,18 @@ async fn windmill_main() -> anyhow::Result<()> { if !skip_migration { // migration code to avoid break - migration_handle = windmill_api::migrate_db(&db).await?; + migration_handle = windmill_api::migrate_db(&db, killpill_rx.resubscribe()).await?; } else { tracing::info!("SKIP_MIGRATION set, skipping db migration...") } } } + if killpill_rx.try_recv().is_ok() { + tracing::info!("Received early killpill, aborting startup"); + return Ok(()); + } + let worker_mode = num_workers > 0; let conn = if mode == Mode::Agent { @@ -484,14 +503,6 @@ async fn windmill_main() -> anyhow::Result<()> { Connection::Sql(db) }; - let (killpill_tx, mut killpill_rx) = KillpillSender::new(2); - let mut monitor_killpill_rx = killpill_tx.subscribe(); - let (killpill_phase2_tx, _killpill_phase2_rx) = tokio::sync::broadcast::channel::<()>(2); - let server_killpill_rx = killpill_phase2_tx.subscribe(); - - let shutdown_signal = - windmill_common::shutdown_signal(killpill_tx.clone(), killpill_tx.subscribe()); - #[cfg(feature = "enterprise")] tracing::info!( " @@ -763,7 +774,7 @@ Windmill Community Edition {GIT_VERSION} conn: if i == 0 || mode != Mode::Agent { conn.clone() } else { - Connection::Http(build_agent_http_client(&suffix)) + Connection::Http(build_agent_http_client(&suffix, None, None)) }, worker_name: worker_name_with_suffix( mode == Mode::Agent, @@ -910,6 +921,8 @@ Windmill Community Edition {GIT_VERSION} } } &"flow" => { + let dynamic_input_key = windmill_common::jobs::generate_dynamic_input_key(workspace_id, path); + windmill_common::DYNAMIC_INPUT_CACHE.remove(&dynamic_input_key); windmill_common::FLOW_VERSION_CACHE.remove(&key); }, _ => { @@ -1043,6 +1056,12 @@ Windmill Community Edition {GIT_VERSION} NUGET_CONFIG_SETTING => { reload_nuget_config_setting(&conn).await }, + POWERSHELL_REPO_URL_SETTING => { + reload_powershell_repo_url_setting(&conn).await + }, + POWERSHELL_REPO_PAT_SETTING => { + reload_powershell_repo_pat_setting(&conn).await + }, MAVEN_REPOS_SETTING => { reload_maven_repos_setting(&conn).await }, @@ -1052,6 +1071,9 @@ Windmill Community Edition {GIT_VERSION} RUBY_REPOS_SETTING => { reload_ruby_repos_setting(&conn).await }, + HUB_API_SECRET_SETTING => { + reload_hub_api_secret_setting(&conn).await + }, KEEP_JOB_DIR_SETTING => { load_keep_job_dir(&conn).await; }, @@ -1256,10 +1278,9 @@ Windmill Community Edition {GIT_VERSION} } if mcp_mode { - futures::try_join!(shutdown_signal, workers_f, server_f)?; + futures::try_join!(workers_f, server_f)?; } else { futures::try_join!( - shutdown_signal, workers_f, monitor_f, server_f, @@ -1284,7 +1305,7 @@ Windmill Community Edition {GIT_VERSION} } } } - Ok(()) + std::process::exit(0); } async fn listen_pg(url: &str) -> Option { diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index a0900da40e..b0818c96cd 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -50,28 +50,28 @@ use windmill_common::{ CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, - JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, - MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, - OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, + JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, + LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, + NUGET_CONFIG_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, + POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, indexer::load_indexer_config, - jobs::QueuedJob, jwt::JWT_SECRET, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_smtp_config, tracing_init::JSON_FMT, users::truncate_token, - utils::{empty_as_none, now_from_db, rd_string, report_critical_error, Mode}, + utils::{empty_as_none, now_from_db, rd_string, report_critical_error, Mode, HUB_API_SECRET}, worker::{ load_env_vars, load_init_bash_from_env, load_periodic_bash_script_from_env, load_periodic_bash_script_interval_from_env, load_whitelist_env_vars_from_env, load_worker_config, reload_custom_tags_setting, store_pull_query, - store_suspended_pull_query, update_min_version, Connection, WorkerConfig, - DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, - SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP, + store_suspended_pull_query, Connection, WorkerConfig, DEFAULT_TAGS_PER_WORKSPACE, + DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, TMP_DIR, + WORKER_CONFIG, WORKER_GROUP, }, KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, @@ -79,11 +79,12 @@ use windmill_common::{ OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS, }; use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING}; -use windmill_queue::{cancel_job, MiniPulledJob, SameWorkerPayload}; +use windmill_queue::{SameWorkerPayload, cancel_job, get_queued_job_v2}; use windmill_worker::{ handle_job_error, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, - NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, + NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, POWERSHELL_REPO_PAT, + POWERSHELL_REPO_URL, }; #[cfg(feature = "parquet")] @@ -283,6 +284,8 @@ pub async fn initial_load( reload_smtp_config(db).await; } + reload_hub_api_secret_setting(&conn).await; + if server_mode { reload_retention_period_setting(&conn).await; reload_request_size(&conn).await; @@ -298,6 +301,8 @@ pub async fn initial_load( reload_bunfig_install_scopes_setting(&conn).await; reload_instance_python_version_setting(&conn).await; reload_nuget_config_setting(&conn).await; + reload_powershell_repo_url_setting(&conn).await; + reload_powershell_repo_pat_setting(&conn).await; reload_maven_repos_setting(&conn).await; reload_no_default_maven_setting(&conn).await; reload_ruby_repos_setting(&conn).await; @@ -867,6 +872,16 @@ pub async fn delete_expired_items(db: &DB) -> () { tracing::error!("Error deleting audit log on CE: {:?}", e); } + if let Err(e) = sqlx::query_scalar!( + "DELETE FROM autoscaling_event WHERE applied_at <= now() - ($1::bigint::text || ' s')::interval", + 30 * 24 * 60 * 60, // 30 days + ) + .fetch_all(db) + .await + { + tracing::error!("Error deleting autoscaling event on CE: {:?}", e); + } + match sqlx::query_scalar!( "DELETE FROM agent_token_blacklist WHERE expires_at <= now() RETURNING token", ) @@ -950,6 +965,17 @@ pub async fn delete_expired_items(db: &DB) -> () { { tracing::error!("Error deleting job: {:?}", e); } + + // should already be deleted but just in case + if let Err(e) = sqlx::query!( + "DELETE FROM job_result_stream_v2 WHERE job_id = ANY($1)", + &deleted_jobs + ) + .execute(&mut *tx) + .await + { + tracing::error!("Error deleting job result stream: {:?}", e); + } } } Err(e) => { @@ -1104,6 +1130,26 @@ pub async fn reload_nuget_config_setting(conn: &Connection) { .await; } +pub async fn reload_powershell_repo_url_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + POWERSHELL_REPO_URL_SETTING, + "POWERSHELL_REPO_URL", + POWERSHELL_REPO_URL.clone(), + ) + .await; +} + +pub async fn reload_powershell_repo_pat_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + POWERSHELL_REPO_PAT_SETTING, + "POWERSHELL_REPO_PAT", + POWERSHELL_REPO_PAT.clone(), + ) + .await; +} + pub async fn reload_maven_repos_setting(conn: &Connection) { reload_option_setting_with_tracing( conn, @@ -1140,6 +1186,16 @@ pub async fn reload_ruby_repos_setting(conn: &Connection) { .await; } +pub async fn reload_hub_api_secret_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + HUB_API_SECRET_SETTING, + "HUB_API_SECRET", + HUB_API_SECRET.clone(), + ) + .await; +} + pub async fn reload_retention_period_setting(conn: &Connection) { if let Err(e) = reload_setting( conn, @@ -1475,7 +1531,7 @@ pub struct MonitorIteration { impl MonitorIteration { pub fn should_run(&self, period: u8) -> bool { - self.iter % (period as u64) == self.rd_shift as u64 + (self.iter + self.rd_shift as u64) % (period as u64) == 0 } } @@ -1531,6 +1587,17 @@ pub async fn monitor_db( } } }; + + let cleanup_debounce_keys_f = async { + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(10) { + if let Some(db) = conn.as_sql() { + if let Err(e) = cleanup_debounce_orphaned_keys(&db).await { + tracing::error!("Error cleaning up debounce keys: {:?}", e); + } + } + } + }; + // run every hour (60 minutes / 30 seconds = 120) let cleanup_worker_group_stats_f = async { if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(120) { @@ -1632,7 +1699,8 @@ pub async fn monitor_db( }; let update_min_worker_version_f = async { - update_min_version(conn).await; + #[cfg(not(feature = "test_job_debouncing"))] + windmill_common::worker::update_min_version(conn).await; }; join!( @@ -1649,12 +1717,13 @@ pub async fn monitor_db( update_min_worker_version_f, cleanup_concurrency_counters_f, cleanup_concurrency_counters_empty_keys_f, + cleanup_debounce_keys_f, cleanup_worker_group_stats_f, ); } async fn vacuuming_tables(db: &Pool) -> error::Result<()> { - sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream, job_stats, job_logs, concurrency_key, log_file, metrics") + sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, concurrency_key, log_file, metrics") .execute(db) .await?; Ok(()) @@ -1977,7 +2046,7 @@ async fn cancel_stale_job( const RESTART_LIMIT: i32 = 3; -async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker_name: &str) { +async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_name: &str) { let mut zombie_jobs_uuid_restart_limit_reached = vec![]; if *RESTART_ZOMBIE_JOBS { @@ -1990,7 +2059,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker LEFT JOIN zombie_job_counter zjc ON zjc.job_id = q.id WHERE ping < now() - ($1 || ' seconds')::interval AND running = true - AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow') + AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') AND same_worker = false AND (zjc.counter IS NULL OR zjc.counter <= $2) FOR UPDATE of q SKIP LOCKED @@ -2104,7 +2173,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker let same_worker_timeout_jobs = { let long_same_worker_jobs = sqlx::query!( "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval - AND running = true AND ping IS NULL AND same_worker = true AND worker IS NOT NULL GROUP BY worker", + AND running = true AND (ping IS NULL OR ping < now() - ('60 seconds')::interval) AND same_worker = true AND worker IS NOT NULL GROUP BY worker", ) .fetch_all(db) .await @@ -2150,24 +2219,18 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker ); } - let jobs = sqlx::query_as::<_, QueuedJob>( - "SELECT *, null as workflow_as_code_status FROM v2_as_queue WHERE id = ANY($1)", - ) - .bind(&timeouts[..]) - .fetch_all(db) - .await - .map_err(|e| tracing::error!("Error fetching same worker jobs: {:?}", e)) - .unwrap_or_default(); - jobs + timeouts }; let non_restartable_jobs = if *RESTART_ZOMBIE_JOBS { vec![] } else { - sqlx::query_as::<_, QueuedJob>("SELECT *, null as workflow_as_code_status FROM v2_as_queue WHERE last_ping < now() - ($1 || ' seconds')::interval - AND running = true AND job_kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow') AND same_worker = false") - .bind(ZOMBIE_JOB_TIMEOUT.as_str()) + sqlx::query_scalar!("SELECT j.id + FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id) + WHERE r.ping < now() - ($1 || ' seconds')::interval + AND q.running = true AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') AND j.same_worker = false", + ZOMBIE_JOB_TIMEOUT.as_str()) .fetch_all(db) .await .ok() @@ -2190,14 +2253,6 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker } } - let zombie_jobs_restart_limit_reached = sqlx::query_as::<_, QueuedJob>( - "SELECT *, null as workflow_as_code_status FROM v2_as_queue WHERE id = ANY($1)", - ) - .bind(&zombie_jobs_uuid_restart_limit_reached[..]) - .fetch_all(db) - .await - .ok() - .unwrap_or_else(|| vec![]); let timeouts = non_restartable_jobs .into_iter() @@ -2208,7 +2263,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker .map(|x| (x, ErrorMessage::SameWorker)), ) .chain( - zombie_jobs_restart_limit_reached + zombie_jobs_uuid_restart_limit_reached .into_iter() .map(|x| (x, ErrorMessage::RestartLimit)), ) @@ -2219,7 +2274,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _); } - for (job, error_kind) in timeouts { + for (job_id, error_kind) in timeouts { // since the job is unrecoverable, the same worker queue should never be sent anything let (same_worker_tx_never_used, _same_worker_rx_never_used) = mpsc::channel::(1); @@ -2228,6 +2283,12 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker let (send_result_never_used, _send_result_rx_never_used) = JobCompletedSender::new_never_used(); + let job = get_queued_job_v2(db, &job_id).await; + if let Err(e) = job { + tracing::error!("Error getting queued job: {:?}", e); + continue; + } + if let Some(job) = job.unwrap() { let label = if job.permissioned_as != format!("u/{}", job.created_by) && job.permissioned_as != job.created_by { @@ -2241,7 +2302,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker &job.permissioned_as, &label, *SCRIPT_TOKEN_EXPIRY, - &job.email, + &job.permissioned_as_email, &job.id, None, Some(format!("handle_zombie_jobs")), @@ -2256,32 +2317,32 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker None, ); - let last_ping = job.last_ping.clone(); let error_message = format!( - "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {}, reason: {:?})", - last_ping - .map(|x| x.to_string()) - .unwrap_or_else(|| "no ping".to_string()), + "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {}, reason: {:?}).\nThis likely means that the job died on worker {}, OOM are a common reason for worker crashes.\nCheck the workers around the time of the last ping and the exit code if any.", + job.last_ping.unwrap_or_default(), *ZOMBIE_JOB_TIMEOUT, - error_kind.to_string() + error_kind.to_string(), + job.worker.clone().unwrap_or_default(), ); + let memory_peak = job.memory_peak.unwrap_or(0); let _ = handle_job_error( db, &client, - &MiniPulledJob::from(&job), - 0, + &windmill_queue::MiniCompletedJob::from(job), + memory_peak, None, error::Error::ExecutionErr(error_message), true, Some(&same_worker_tx_never_used), "", - worker_name, + node_name, send_result_never_used, #[cfg(feature = "benchmark")] &mut windmill_common::bench::BenchmarkIter::new(), ) .await; } + } } async fn cleanup_concurrency_counters_orphaned_keys(db: &DB) -> error::Result<()> { @@ -2358,6 +2419,7 @@ async fn cleanup_concurrency_counters_empty_keys(db: &DB) -> error::Result<()> { WITH rows_to_delete AS ( SELECT concurrency_id FROM concurrency_counter + WHERE job_uuids = '{}'::jsonb FOR UPDATE SKIP LOCKED ) @@ -2386,13 +2448,13 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { let flows = sqlx::query!( r#" SELECT - id AS "id!", workspace_id AS "workspace_id!", parent_job, is_flow_step, - flow_status AS "flow_status: Box", last_ping, same_worker - FROM v2_as_queue - WHERE running = true AND suspend = 0 AND suspend_until IS null AND scheduled_for <= now() - AND (job_kind = 'flow' OR job_kind = 'flowpreview' OR job_kind = 'flownode') - AND last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval - AND canceled = false + j.id AS "id!", j.workspace_id AS "workspace_id!", j.parent_job, j.flow_step_id IS NOT NULL AS "is_flow_step?", + COALESCE(s.flow_status, s.workflow_as_code_status) AS "flow_status: Box", r.ping AS last_ping, j.same_worker AS "same_worker?" + FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id) + WHERE q.running = true AND q.suspend = 0 AND q.suspend_until IS null AND q.scheduled_for <= now() + AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode') + AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval + AND q.canceled_by IS NULL "#, FLOW_ZOMBIE_TRANSITION_TIMEOUT.as_str() @@ -2667,7 +2729,7 @@ pub async fn reload_critical_alerts_on_db_oversize(conn: &DB) -> error::Result<( async fn generate_and_save_jwt_secret(db: &DB) -> error::Result { let secret = rd_string(32); sqlx::query!( - "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = $2", + "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value", JWT_SECRET_SETTING, serde_json::to_value(&secret).unwrap() ).execute(db).await?; @@ -2695,3 +2757,29 @@ pub async fn reload_jwt_secret_setting(db: &DB) -> error::Result<()> { Ok(()) } + +async fn cleanup_debounce_orphaned_keys(db: &DB) -> error::Result<()> { + let result = sqlx::query!( + " +DELETE FROM debounce_key +WHERE job_id NOT IN (SELECT id FROM v2_job_queue) +RETURNING key,job_id + ", + ) + .fetch_all(db) + .await?; + + tracing::debug!("Cleaning up debounce keys"); + + if result.len() > 0 { + tracing::info!("Cleaned up {} debounce keys", result.len()); + for row in result { + tracing::info!( + "Debounce key cleaned up: key: {}, job_id: {:?}", + row.key, + row.job_id + ); + } + } + Ok(()) +} diff --git a/backend/tests/common/mod.rs b/backend/tests/common/mod.rs index 18416d7cc2..40a9980a96 100644 --- a/backend/tests/common/mod.rs +++ b/backend/tests/common/mod.rs @@ -11,9 +11,41 @@ use uuid::Uuid; use windmill_api_client::types::NewScript; #[cfg(feature = "python")] use windmill_common::flow_status::FlowStatusModule; -use windmill_common::{jobs::{JobKind, JobPayload, RawCode}, jwt::JWT_SECRET, scripts::{ ScriptHash, ScriptLang}, worker::WORKER_CONFIG, KillpillSender}; +use windmill_common::{ + jobs::{JobKind, JobPayload, RawCode}, + jwt::JWT_SECRET, + scripts::{ScriptHash, ScriptLang}, + worker::{Connection, WORKER_CONFIG}, + KillpillSender, +}; use windmill_queue::PushIsolationLevel; +pub async fn init_client(db: Pool) -> (windmill_api_client::Client, u16, ApiServer) { + initialize_tracing().await; + let server = ApiServer::start(db).await.unwrap(); + let port = server.addr.port(); + let client = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN".to_string(), + ); + (client, port, server) +} + +pub async fn init_client_agent_mode( + db: Pool, +) -> (windmill_api_client::Client, u16, ApiServer) { + initialize_tracing().await; + set_jwt_secret().await; + + let server = ApiServer::start_agent_mode(db).await.unwrap(); + let port = server.addr.port(); + let client = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN".to_string(), + ); + (client, port, server) +} + /// it's important this is unique between tests as there is one prometheus registry and /// run_worker shouldn't register the same metric with the same worker name more than once. /// @@ -49,6 +81,14 @@ pub struct ApiServer { impl ApiServer { pub async fn start(db: Pool) -> anyhow::Result { + Self::start_inner(db, false).await + } + + pub async fn start_agent_mode(db: Pool) -> anyhow::Result { + Self::start_inner(db, true).await + } + + async fn start_inner(db: Pool, agent_mode: bool) -> anyhow::Result { let (tx, rx) = tokio::sync::broadcast::channel::<()>(1); let sock = tokio::net::TcpListener::bind("127.0.0.1:0") @@ -69,17 +109,20 @@ impl ApiServer { addr, rx, port_tx, - false, + agent_mode, false, format!("http://localhost:{}", addr.port()), Some(name.clone()), )); tracing::info!("waiting for server port for name={name}"); - _port_rx.await.map_err(|e| { + if let Err(e) = _port_rx.await { tracing::error!("failed to receive port for name={name}: {e}"); - anyhow::anyhow!("failed to receive port for name={name}: {e}") - })?; + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + return Err(anyhow::anyhow!( + "failed to receive port for name={name}: {e}" + )); + } // clear the cache between tests windmill_common::cache::clear(); @@ -96,15 +139,17 @@ impl ApiServer { } } - +#[derive(Debug, Clone)] pub struct RunJob { pub payload: JobPayload, pub args: serde_json::Map, + pub debounce_job_id_o: Option, + pub scheduled_for_o: Option>, } impl From for RunJob { fn from(payload: JobPayload) -> Self { - Self { payload, args: Default::default() } + Self { payload, args: Default::default(), debounce_job_id_o: None, scheduled_for_o: None } } } @@ -114,8 +159,21 @@ impl RunJob { self } + pub fn push_arg_debounce_job_id_o(mut self, job_id: Option) -> Self { + self.debounce_job_id_o = job_id; + self + } + + pub fn push_arg_scheduled_for_o( + mut self, + scheduled_for_o: Option>, + ) -> Self { + self.scheduled_for_o = scheduled_for_o; + self + } + pub async fn push(self, db: &Pool) -> Uuid { - let RunJob { payload, args } = self; + let RunJob { payload, args, debounce_job_id_o, scheduled_for_o } = self; let mut hm_args = std::collections::HashMap::new(); for (k, v) in args { hm_args.insert(k, windmill_common::worker::to_raw_value(&v)); @@ -123,7 +181,7 @@ impl RunJob { let tx = PushIsolationLevel::IsolatedRoot(db.clone()); let (uuid, tx) = windmill_queue::push( - &db, + db, tx, "test-workspace", payload, @@ -132,7 +190,7 @@ impl RunJob { /* email */ "test@windmill.dev", /* permissioned_as */ "u/test-user".to_string(), /* token_prefix */ None, - /* scheduled_for_o */ None, + scheduled_for_o, /* schedule_path */ None, /* parent_job */ None, /* root job */ None, @@ -148,6 +206,8 @@ impl RunJob { None, None, false, + None, + debounce_job_id_o, ) .await .expect("push has to succeed"); @@ -157,48 +217,81 @@ impl RunJob { } /// push the job, spawn a worker, wait until the job is in completed_job - pub async fn run_until_complete(self, db: &Pool, port: u16) -> CompletedJob { + pub async fn run_until_complete( + self, + db: &Pool, + agent_mode: bool, + port: u16, + ) -> CompletedJob { let uuid = self.push(db).await; let listener = listen_for_completed_jobs(db).await; - in_test_worker(db, listener.find(&uuid), port).await; - let r = completed_job(uuid, db).await; - r + + let conn = match agent_mode { + false => Connection::Sql(db.clone()), + #[cfg(all(feature = "private", feature = "agent_worker_server"))] + true => testing_http_connection(port).await, + #[cfg(not(all(feature = "private", feature = "agent_worker_server")))] + true => { + panic!("to use agent worker test, you need to enable 'agent_worker_server' feature") + } + }; + + in_test_worker(conn, listener.find(&uuid), port).await; + + completed_job(uuid, db).await } /// push the job, spawn a worker, wait until the job is in completed_job pub async fn run_until_complete_with>( self, db: &Pool, + agent_mode: bool, port: u16, test: impl Fn(Uuid) -> F, ) -> CompletedJob { let uuid = self.push(db).await; let listener = listen_for_completed_jobs(db).await; test(uuid).await; - in_test_worker(db, listener.find(&uuid), port).await; - let r = completed_job(uuid, db).await; - r + + let conn = match agent_mode { + false => Connection::Sql(db.clone()), + #[cfg(all(feature = "private", feature = "agent_worker_server"))] + true => testing_http_connection(port).await, + #[cfg(not(all(feature = "private", feature = "agent_worker_server")))] + true => { + panic!("to use agent worker test, you need to enable 'agent_worker_server' feature") + } + }; + + in_test_worker(conn, listener.find(&uuid), port).await; + + completed_job(uuid, db).await } } pub async fn run_job_in_new_worker_until_complete( db: &Pool, + agent_mode: bool, job: JobPayload, port: u16, ) -> CompletedJob { - RunJob::from(job).run_until_complete(db, port).await + RunJob::from(job) + .run_until_complete(db, agent_mode, port) + .await } /// Start a worker with a timeout and run a future, until the worker quits or we time out. /// /// Cleans up the worker before resolving. pub async fn in_test_worker( - db: &Pool, + // db: &Pool, + // If set to http, worker will be started in agent mode. + conn: impl Into, inner: Fut, port: u16, ) -> ::Output { set_jwt_secret().await; - let (quit, worker) = spawn_test_worker(db, port); + let (quit, worker) = spawn_test_worker(&conn.into(), port); let worker = tokio::time::timeout(std::time::Duration::from_secs(60), worker); tokio::pin!(worker); @@ -223,7 +316,7 @@ pub async fn in_test_worker( } pub fn spawn_test_worker( - db: &Pool, + conn: &Connection, port: u16, ) -> (KillpillSender, tokio::task::JoinHandle<()>) { std::fs::DirBuilder::new() @@ -232,26 +325,26 @@ pub fn spawn_test_worker( .expect("could not create initial worker dir"); let (tx, rx) = KillpillSender::new(1); - let db = db.to_owned(); let worker_instance: &str = "test worker instance"; let worker_name: String = next_worker_name(); let ip: &str = Default::default(); + let conn = conn.to_owned(); let tx2 = tx.clone(); let future = async move { let base_internal_url = format!("http://localhost:{}", port); { let mut wc = WORKER_CONFIG.write().await; - (*wc).worker_tags = windmill_common::worker::DEFAULT_TAGS.clone(); - (*wc).priority_tags_sorted = vec![windmill_common::worker::PriorityTags { + wc.worker_tags = windmill_common::worker::DEFAULT_TAGS.clone(); + wc.priority_tags_sorted = vec![windmill_common::worker::PriorityTags { priority: 0, - tags: (*wc).worker_tags.clone(), + tags: wc.worker_tags.clone(), }]; windmill_common::worker::store_suspended_pull_query(&wc).await; windmill_common::worker::store_pull_query(&wc).await; } windmill_worker::run_worker( - &db.into(), + &conn, worker_instance, worker_name, 1, @@ -283,22 +376,35 @@ pub async fn listen_for_uuid_on( let mut listener = PgListener::connect_with(db).await.unwrap(); listener.listen(channel).await.unwrap(); - Box::pin(futures::stream::unfold(listener, |mut listener| async move { - let uuid = listener - .try_recv() - .await - .unwrap() - .expect("lost database connection") - .payload() - .parse::() - .expect("invalid uuid"); - Some((uuid, listener)) - })) + Box::pin(futures::stream::unfold( + listener, + |mut listener| async move { + let uuid = listener + .try_recv() + .await + .unwrap() + .expect("lost database connection") + .payload() + .parse::() + .expect("invalid uuid"); + Some((uuid, listener)) + }, + )) } pub async fn completed_job(uuid: Uuid, db: &Pool) -> CompletedJob { sqlx::query_as::<_, CompletedJob>( - "SELECT *, result->'wm_labels' as labels FROM v2_as_completed_job WHERE id = $1", + "SELECT j.id, j.workspace_id, j.parent_job, j.created_by, j.created_at, c.duration_ms, + c.status = 'success' OR c.status = 'skipped' AS success, j.runnable_id AS script_hash, j.runnable_path AS script_path, + j.args, c.result, FALSE AS deleted, j.raw_code, c.status = 'canceled' AS canceled, + c.canceled_by, c.canceled_reason, j.kind AS job_kind, + CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END AS schedule_path, + j.permissioned_as, COALESCE(c.flow_status, c.workflow_as_code_status) AS flow_status, j.raw_flow, + j.flow_step_id IS NOT NULL AS is_flow_step, j.script_lang AS language, c.started_at, + c.status = 'skipped' AS is_skipped, j.raw_lock, j.permissioned_as_email AS email, j.visible_to_owner, + c.memory_peak AS mem_peak, j.tag, j.priority, NULL::TEXT AS logs, c.result_columns, + j.script_entrypoint_override, j.preprocessed, c.result->'wm_labels' as labels + FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.id = $1", ) .bind(uuid) .fetch_one(db) @@ -320,7 +426,6 @@ pub trait StreamFind: futures::Stream + Unpin + Sized { impl StreamFind for T {} - #[cfg(feature = "python")] pub fn get_module(cjob: &CompletedJob, id: &str) -> Option { cjob.flow_status.clone().and_then(|fs| { @@ -338,7 +443,7 @@ fn find_module_in_vec(modules: Vec, id: &str) -> Option () { +pub async fn set_jwt_secret() { let secret = "mytestsecret".to_string(); let mut l = JWT_SECRET.write().await; *l = secret; @@ -428,7 +533,8 @@ pub async fn assert_lockfile( .create_script( "test-workspace", &NewScript { - language: windmill_api_client::types::ScriptLang::from_str(language.as_str()).unwrap(), + language: windmill_api_client::types::ScriptLang::from_str(language.as_str()) + .unwrap(), content: script_content, path: "f/system/test_import".to_string(), concurrent_limit: None, @@ -463,10 +569,10 @@ pub async fn assert_lockfile( .await .unwrap(); - let mut completed = listen_for_completed_jobs(&db).await; + let mut completed = listen_for_completed_jobs(db).await; let db2 = db.clone(); in_test_worker( - &db, + db, async move { completed.next().await; // deployed script @@ -506,8 +612,6 @@ pub async fn assert_lockfile( Ok(()) } - - pub async fn run_deployed_relative_imports( db: &Pool, script_content: String, @@ -525,7 +629,8 @@ pub async fn run_deployed_relative_imports( .create_script( "test-workspace", &NewScript { - language: windmill_api_client::types::ScriptLang::from_str(language.as_str()).unwrap(), + language: windmill_api_client::types::ScriptLang::from_str(language.as_str()) + .unwrap(), content: script_content, path: "f/system/test_import".to_string(), concurrent_limit: None, @@ -560,10 +665,10 @@ pub async fn run_deployed_relative_imports( .await .unwrap(); - let mut completed = listen_for_completed_jobs(&db).await; + let mut completed = listen_for_completed_jobs(db).await; let db2 = db.clone(); in_test_worker( - &db, + db, async move { completed.next().await; // deployed script @@ -586,6 +691,8 @@ pub async fn run_deployed_relative_imports( language, priority: None, apply_preprocessor: false, + custom_debounce_key: None, + debounce_delay_s: None, }) .push(&db2) .await; @@ -620,10 +727,10 @@ pub async fn run_preview_relative_imports( let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); - let mut completed = listen_for_completed_jobs(&db).await; + let mut completed = listen_for_completed_jobs(db).await; let db2 = db.clone(); in_test_worker( - &db, + db.clone(), async move { let job = RunJob::from(JobPayload::Code(RawCode { hash: None, @@ -636,6 +743,8 @@ pub async fn run_preview_relative_imports( concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, + custom_debounce_key: None, + debounce_delay_s: None, })) .push(&db2) .await; @@ -659,4 +768,45 @@ pub async fn run_preview_relative_imports( .await; Ok(()) -} \ No newline at end of file +} + +#[cfg(all(feature = "private", feature = "agent_worker_server"))] +pub async fn testing_http_connection(port: u16) -> Connection { + let suffix = windmill_common::utils::create_default_worker_suffix("test-agent-worker"); + Connection::Http(windmill_common::agent_workers::build_agent_http_client( + &suffix, + Some(format!( + "{}{}", + windmill_common::agent_workers::AGENT_JWT_PREFIX, + windmill_common::jwt::encode_with_internal_secret( + windmill_api::agent_workers_ee::AgentAuth { + worker_group: "testing-agent".to_owned(), + suffix: Some(suffix.clone()), + tags: vec!["flow".into(), "python3".into(), "dependency".into()], + exp: Some(usize::MAX), + } + ) + .await + .expect("JWT token to be created") + )), + Some(format!("http://localhost:{port}")), + )) +} + +/// IMPORTANT!: +/// Do not run parallel in tests! +/// +/// No tests can run this at the same time, will result into conflicts!!! +pub async fn rebuild_dmap(client: &windmill_api_client::Client) -> bool { + client + .client() + .post(format!( + "{}/w/test-workspace/workspaces/rebuild_dependency_map", + client.baseurl() + )) + .send() + .await + .unwrap() + .status() + .is_success() +} diff --git a/backend/tests/fixtures/dependency_map.sql b/backend/tests/fixtures/dependency_map.sql new file mode 100644 index 0000000000..c2caf110be --- /dev/null +++ b/backend/tests/fixtures/dependency_map.sql @@ -0,0 +1,118 @@ +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +def main(): + return "f/rel/leaf_1" +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/rel/leaf_1', 333400, 'python3', ''); +-- Padded Hex: 0000000000051658 + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +def main(): + return "f/rel/leaf_2" +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/rel/leaf_2', 333401, 'python3', ''); +-- Padded Hex: 0000000000051659 + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +from f.rel.leaf_1 import main as lf_1; + +def main(): + return lf_1(); +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/rel/branch', 333402, 'python3', ''); +-- Padded Hex: 000000000005165A + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +from f.rel.branch import main as br; +from f.rel.leaf_1 import main as lf_1; +from f.rel.leaf_2 import main as lf_2; + +def main(): + return [br(), lf_1(), lf_2]; +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/rel/root_script', 333403, 'python3', ''); +-- Padded Hex: 000000000005165B + +INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( +'test-workspace', +'', +'', +'f/rel/root_flow', +'{1443253234253454}', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object","order":[]}', +$tag${"modules":[{"id":"nstep1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}},{"id":"nstep2","value":{"type":"branchall","branches":[{"expr":"","modules":[{"id":"nstep2_1","value":{"lock":"{\n \"dependencies\": {}\n}\n//bun.lock\n","type":"rawscript","assets":[],"content":"// import * as wmill from \"windmill-client\"\n\nexport async function main(x: string) {\n return x\n}\n","language":"bun","input_transforms":{"x":{"type":"static","value":""}}}}],"parallel":true,"skip_failure":false},{"expr":"false","modules":[{"id":"nstep2_2","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"summary":"","parallel":true,"skip_failure":false}],"parallel":true},"summary":""},{"id":"nstep3","value":{"type":"branchone","default":[{"id":"nstep3_2","value":{"lock":"{\n \"dependencies\": {}\n}\n//bun.lock\n","type":"rawscript","assets":[],"content":"// import * as wmill from \"windmill-client\"\n\nexport async function main(x: string) {\n return x\n}\n","language":"bun","input_transforms":{"x":{"type":"static","value":""}}}}],"branches":[{"expr":"false","modules":[{"id":"nstep3_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"def main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"summary":"","parallel":true,"skip_failure":true}]},"summary":""},{"id":"nstep4","value":{"type":"whileloopflow","modules":[{"id":"nstep4_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\n\ndef check():\n return [br()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"skip_failures":false}},{"id":"nstep5","value":{"type":"forloopflow","modules":[{"id":"nstep5_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"iterator":{"expr":"['dynamic or static array']","type":"javascript"},"parallel":false,"skip_failures":true}},{"id":"nstep_ai","value":{"type":"aiagent","tools":[{"id":"qtool1","value":{"type":"rawscript","content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\ndef main(x):\n return x\n","language":"python3","input_transforms":{"x":{"type":"static"}}},"summary":"tool"}],"input_transforms":{"image":{"type":"static"},"provider":{"type":"static","value":{"kind":"openai"}},"output_type":{"type":"static","value":"text"},"temperature":{"type":"static"},"user_message":{"type":"static","value":""},"output_schema":{"type":"static"},"system_prompt":{"type":"static","value":""},"max_completion_tokens":{"type":"static"}}},"continue_on_error":false}],"failure_module":{"id":"failure","value":{"type":"rawscript","content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\nimport os\n\ndef main(message: str, name: str, step_id: str):\n flow_id = os.environ.get(\"WM_ROOT_FLOW_JOB_ID\")\n print(\"message\", message)\n print(\"name\", name)\n print(\"step_id\", step_id)\n return { \"message\": message, \"flow_id\": flow_id, \"step_id\": step_id, \"recover\": False }","language":"python3","input_transforms":{"name":{"expr":"error.name","type":"javascript"},"message":{"expr":"error.message","type":"javascript"},"step_id":{"expr":"error.step_id","type":"javascript"}}}},"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef preprocessor(event):\n return {\n # return the args to be passed to the runnable\n }\n","language":"python3","input_transforms":{"event":{"type":"static"}}}}}$tag$, +'system' +); + +INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_by) VALUES ( +1443253234253454, +'test-workspace', +'f/rel/root_flow', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object","order":[]}', +$tag${"modules":[{"id":"nstep1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}},{"id":"nstep2","value":{"type":"branchall","branches":[{"expr":"","modules":[{"id":"nstep2_1","value":{"lock":"{\n \"dependencies\": {}\n}\n//bun.lock\n","type":"rawscript","assets":[],"content":"// import * as wmill from \"windmill-client\"\n\nexport async function main(x: string) {\n return x\n}\n","language":"bun","input_transforms":{"x":{"type":"static","value":""}}}}],"parallel":true,"skip_failure":false},{"expr":"false","modules":[{"id":"nstep2_2","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"summary":"","parallel":true,"skip_failure":false}],"parallel":true},"summary":""},{"id":"nstep3","value":{"type":"branchone","default":[{"id":"nstep3_2","value":{"lock":"{\n \"dependencies\": {}\n}\n//bun.lock\n","type":"rawscript","assets":[],"content":"// import * as wmill from \"windmill-client\"\n\nexport async function main(x: string) {\n return x\n}\n","language":"bun","input_transforms":{"x":{"type":"static","value":""}}}}],"branches":[{"expr":"false","modules":[{"id":"nstep3_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"def main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"summary":"","parallel":true,"skip_failure":true}]},"summary":""},{"id":"nstep4","value":{"type":"whileloopflow","modules":[{"id":"nstep4_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\n\ndef check():\n return [br()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"skip_failures":false}},{"id":"nstep5","value":{"type":"forloopflow","modules":[{"id":"nstep5_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"iterator":{"expr":"['dynamic or static array']","type":"javascript"},"parallel":false,"skip_failures":true}},{"id":"nstep_ai","value":{"type":"aiagent","tools":[{"id":"qtool1","value":{"type":"rawscript","content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\ndef main(x):\n return x\n","language":"python3","input_transforms":{"x":{"type":"static"}}},"summary":"tool"}],"input_transforms":{"image":{"type":"static"},"provider":{"type":"static","value":{"kind":"openai"}},"output_type":{"type":"static","value":"text"},"temperature":{"type":"static"},"user_message":{"type":"static","value":""},"output_schema":{"type":"static"},"system_prompt":{"type":"static","value":""},"max_completion_tokens":{"type":"static"}}},"continue_on_error":false}],"failure_module":{"id":"failure","value":{"type":"rawscript","content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\nimport os\n\ndef main(message: str, name: str, step_id: str):\n flow_id = os.environ.get(\"WM_ROOT_FLOW_JOB_ID\")\n print(\"message\", message)\n print(\"name\", name)\n print(\"step_id\", step_id)\n return { \"message\": message, \"flow_id\": flow_id, \"step_id\": step_id, \"recover\": False }","language":"python3","input_transforms":{"name":{"expr":"error.name","type":"javascript"},"message":{"expr":"error.message","type":"javascript"},"step_id":{"expr":"error.step_id","type":"javascript"}}}},"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef preprocessor(event):\n return {\n # return the args to be passed to the runnable\n }\n","language":"python3","input_transforms":{"event":{"type":"static"}}}}}$tag$, +'system' +); + +INSERT INTO public.app(id, workspace_id, path, versions, policy) VALUES ( +2, +'test-workspace', +'f/rel/root_app', +'{0}', +'{}' +); + +INSERT INTO public.app_version(id, app_id, value, created_by) VALUES ( +0, +2, +$tag${"grid":[{"3":{"h":2,"w":6,"x":0,"y":0,"fixed":true,"fullHeight":false},"12":{"h":2,"w":12,"x":0,"y":0,"fixed":true,"fullHeight":false},"id":"topbar","data":{"id":"topbar","type":"containercomponent","customCss":{"container":{"class":"!p-0","style":""}},"configuration":{},"numberOfSubgrids":1}},{"3":{"h":8,"w":2,"x":0,"y":2,"fixed":false,"fullHeight":false},"12":{"h":2,"w":6,"x":0,"y":2,"fixed":false,"fullHeight":false},"id":"a","data":{"id":"a","type":"containercomponent","customCss":{"container":{"class":"","style":""}},"configuration":{},"numberOfSubgrids":1}},{"3":{"h":1,"w":1,"x":2,"y":2,"fixed":false,"fullHeight":false},"12":{"h":1,"w":2,"x":6,"y":2,"fixed":false,"fullHeight":false},"id":"dontpressmeplz","data":{"id":"dontpressmeplz","type":"buttoncomponent","customCss":{"button":{"class":"","style":""},"container":{"class":"","style":""}},"recomputeIds":[],"configuration":{"size":{"type":"static","value":"xs"},"color":{"type":"static","value":"blue"},"label":{"type":"static","value":"Press me"},"onError":{"type":"oneOf","selected":"errorOverlay","configuration":{"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"errorOverlay":{},"sendErrorToast":{"message":{"type":"static","value":"An error occured"},"appendError":{"type":"static","value":true}}}},"disabled":{"type":"static","value":false},"afterIcon":{"type":"static"},"onSuccess":{"type":"oneOf","selected":"none","configuration":{"none":{},"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"openModal":{"modalId":{"type":"static","value":""}},"sendToast":{"message":{"type":"static","value":""}},"clearFiles":{"id":{"type":"static","value":""}},"closeModal":{"modalId":{"type":"static","value":""}}}},"beforeIcon":{"type":"static"},"fillContainer":{"type":"static","value":false},"triggerOnAppLoad":{"type":"static","value":false},"confirmationModal":{"type":"oneOf","selected":"none","configuration":{"none":{},"confirmationModal":{"title":{"type":"static","value":"Title"},"description":{"type":"static","value":"Are you sure?"},"confirmationText":{"type":"static","value":"Confirm"}}}}},"componentInput":{"type":"runnable","fields":{"x":{"type":"static","value":null,"fieldType":"string"}},"runnable":{"name":"Inline Script","type":"runnableByName","inlineScript":{"path":"u/admin@windmill.dev/newapp/Inline_Script","schema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["x"],"properties":{"x":{"default":null,"description":"","originalType":"string","type":"string"}}},"content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n \ndef main(x: str):\n return x","language":"python3"}},"fieldType":"any","autoRefresh":false,"recomputeOnInputChanged":false},"verticalAlignment":"center","horizontalAlignment":"center"}},{"3":{"h":1,"w":1,"x":2,"y":3,"fixed":false,"fullHeight":false},"12":{"h":1,"w":2,"x":8,"y":2,"fixed":false,"fullHeight":false},"id":"d","data":{"id":"d","type":"checkboxcomponent","customCss":{"text":{"class":"","style":""},"container":{"class":"","style":""}},"recomputeIds":[],"configuration":{"label":{"type":"static","value":"Label"},"disabled":{"type":"static","value":false},"defaultValue":{"type":"static","value":false}},"verticalAlignment":"center","horizontalAlignment":"center"}},{"3":{"h":1,"w":1,"x":2,"y":4,"fixed":false,"fullHeight":false},"12":{"h":1,"w":2,"x":6,"y":3,"fixed":false,"fullHeight":false},"id":"youcanpressme","data":{"id":"youcanpressme","type":"buttoncomponent","customCss":{"button":{"class":"","style":""},"container":{"class":"","style":""}},"recomputeIds":[],"configuration":{"size":{"type":"static","value":"xs"},"color":{"type":"static","value":"blue"},"label":{"type":"static","value":"Press me"},"onError":{"type":"oneOf","selected":"errorOverlay","configuration":{"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"errorOverlay":{},"sendErrorToast":{"message":{"type":"static","value":"An error occured"},"appendError":{"type":"static","value":true}}}},"disabled":{"type":"static","value":false},"afterIcon":{"type":"static"},"onSuccess":{"type":"oneOf","selected":"none","configuration":{"none":{},"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"openModal":{"modalId":{"type":"static","value":""}},"sendToast":{"message":{"type":"static","value":""}},"clearFiles":{"id":{"type":"static","value":""}},"closeModal":{"modalId":{"type":"static","value":""}}}},"beforeIcon":{"type":"static"},"fillContainer":{"type":"static","value":false},"triggerOnAppLoad":{"type":"static","value":false},"confirmationModal":{"type":"oneOf","selected":"none","configuration":{"none":{},"confirmationModal":{"title":{"type":"static","value":"Title"},"description":{"type":"static","value":"Are you sure?"},"confirmationText":{"type":"static","value":"Confirm"}}}}},"componentInput":{"type":"runnable","fields":{"x":{"type":"static","value":null,"fieldType":"string"}},"runnable":{"name":"Inline Script","type":"runnableByName","inlineScript":{"path":"u/admin/easy_to_use_app/Inline_Script","schema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["x"],"properties":{"x":{"default":null,"description":"","originalType":"string","type":"string"}}},"content":"from f.rel.branch import main as br;\n\ndef check():\n return [br()];\n\ndef main(x: str):\n return x","language":"python3"}},"fieldType":"any","autoRefresh":false,"recomputeOnInputChanged":false},"verticalAlignment":"center","horizontalAlignment":"center"}}],"theme":{"path":"f/app_themes/theme_0","type":"path"},"subgrids":{"a-0":[{"3":{"h":1,"w":1,"x":0,"y":0,"fixed":false,"fullHeight":false},"12":{"h":2,"w":5,"x":0,"y":0,"fixed":false,"fullHeight":false},"id":"pressmeplz","data":{"id":"pressmeplz","type":"buttoncomponent","customCss":{"button":{"class":"","style":""},"container":{"class":"","style":""}},"recomputeIds":[],"configuration":{"size":{"type":"static","value":"xs"},"color":{"type":"static","value":"blue"},"label":{"type":"static","value":"Press me"},"onError":{"type":"oneOf","selected":"errorOverlay","configuration":{"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"errorOverlay":{},"sendErrorToast":{"message":{"type":"static","value":"An error occured"},"appendError":{"type":"static","value":true}}}},"disabled":{"type":"static","value":false},"afterIcon":{"type":"static"},"onSuccess":{"type":"oneOf","selected":"none","configuration":{"none":{},"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"openModal":{"modalId":{"type":"static","value":""}},"sendToast":{"message":{"type":"static","value":""}},"clearFiles":{"id":{"type":"static","value":""}},"closeModal":{"modalId":{"type":"static","value":""}}}},"beforeIcon":{"type":"static"},"fillContainer":{"type":"static","value":false},"triggerOnAppLoad":{"type":"static","value":false},"confirmationModal":{"type":"oneOf","selected":"none","configuration":{"none":{},"confirmationModal":{"title":{"type":"static","value":"Title"},"description":{"type":"static","value":"Are you sure?"},"confirmationText":{"type":"static","value":"Confirm"}}}}},"componentInput":{"type":"runnable","fields":{"x":{"type":"static","value":null,"fieldType":"string"}},"runnable":{"name":"Inline Script","type":"runnableByName","inlineScript":{"path":"u/admin@windmill.dev/newapp/Inline_Script","schema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["x"],"properties":{"x":{"default":null,"description":"","originalType":"string","type":"string"}}},"content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\n\ndef main(x: str):\n return x","language":"python3"}},"fieldType":"any","autoRefresh":false,"recomputeOnInputChanged":false},"verticalAlignment":"center","horizontalAlignment":"center"}}],"topbar-0":[{"3":{"h":1,"w":6,"x":0,"y":0,"fixed":false,"fullHeight":false},"12":{"h":1,"w":6,"x":0,"y":0,"fixed":false,"fullHeight":false},"id":"title","data":{"id":"title","type":"textcomponent","customCss":{"text":{"class":"text-xl font-semibold whitespace-nowrap truncate","style":""},"container":{"class":"","style":""}},"configuration":{"style":{"type":"static","value":"Body"},"tooltip":{"expr":"`Author: ${ctx.author}`","type":"evalv2","value":"","fieldType":"text","connections":[{"id":"author","componentId":"ctx"}]},"copyButton":{"type":"static","value":false},"disableNoText":{"type":"static","value":true,"fieldType":"boolean"}},"componentInput":{"eval":"${ctx.summary}","type":"templatev2","fieldType":"template","connections":[{"id":"summary","componentId":"ctx"}]},"verticalAlignment":"center","horizontalAlignment":"left"}},{"3":{"h":1,"w":3,"x":0,"y":1,"fixed":false,"fullHeight":false},"12":{"h":1,"w":6,"x":6,"y":0,"fixed":false,"fullHeight":false},"id":"recomputeall","data":{"id":"recomputeall","type":"recomputeallcomponent","customCss":{"container":{"class":"","style":""}},"menuItems":[],"configuration":{"defaultRefreshInterval":{"type":"static","value":"0"}},"verticalAlignment":"center","horizontalAlignment":"right"}}]},"fullscreen":false,"norefreshbar":false,"hideLegacyTopBar":true,"hiddenInlineScripts":[],"unusedInlineScripts":[],"mobileViewOnSmallerScreens":false}$tag$, +'system' +); + +-- Prebuild dependency_map +-- It would be done by Windmill, but this one is static. +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/branch', 'script', 'f/rel/leaf_1', ''); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_script', 'script', 'f/rel/branch', ''); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_script', 'script', 'f/rel/leaf_1', ''); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_script', 'script', 'f/rel/leaf_2', ''); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_app', 'app', 'f/rel/leaf_2', 'dontpressmeplz'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/leaf_2', 'failure'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/branch', 'nstep1'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/leaf_1', 'nstep1'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/leaf_2', 'nstep1'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/leaf_2', 'nstep2_2'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/branch', 'nstep4_1'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/branch', 'nstep5_1'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/leaf_1', 'nstep5_1'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/leaf_2', 'nstep5_1'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/branch', 'preprocessor'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/leaf_1', 'preprocessor'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/leaf_2', 'preprocessor'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_app', 'app', 'f/rel/branch', 'pressmeplz'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_app', 'app', 'f/rel/leaf_1', 'pressmeplz'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_app', 'app', 'f/rel/leaf_2', 'pressmeplz'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/leaf_2', 'qtool1'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_app', 'app', 'f/rel/branch', 'youcanpressme'); + diff --git a/backend/tests/fixtures/djob_debouncing.sql b/backend/tests/fixtures/djob_debouncing.sql new file mode 100644 index 0000000000..067d46c398 --- /dev/null +++ b/backend/tests/fixtures/djob_debouncing.sql @@ -0,0 +1,242 @@ +-- FLOWS -- +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +'#requirements: +#bottle==0.13.2 +def main(): + pass +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/dre/leaf_left', 333400, 'python3', ''); +-- Padded Hex: 0000000000051658 + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +'#requirements: +#tiny==0.1.3 +def main(): + pass +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/dre/leaf_right', 333403, 'python3', ''); +-- Padded Hex: 000000000005165B + + +INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( +'test-workspace', +'', +'', +'f/dre/flow', +'{1443253234253454}', +'{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {}, + "required": [], + "type": "object" +}', +$tag$ +{ + "modules": [ + { + "id": "a", + "value": { + "lock": "# py: 3.11\n", + "type": "rawscript", + "assets": [], + "content": "import f.dre.leaf_left\n\ndef main():\n pass", + "language": "python3", + "input_transforms": {} + }, + "summary": "leaf Left" + }, + { + "id": "b", + "value": { + "lock": "# py: 3.11\n", + "type": "rawscript", + "assets": [], + "content": "import f.dre.leaf_right\nimport f.dre.leaf_left\n\ndef main():\n pass", + "language": "python3", + "input_transforms": {} + }, + "summary": "leaf Left and Right" + }, + { + "id": "c", + "value": { + "lock": "# py: 3.11\n", + "type": "rawscript", + "assets": [], + "content": "import f.dre.leaf_right\n\ndef main():\n pass", + "language": "python3", + "input_transforms": {} + }, + "summary": "leaf RIght" + } + ] +}$tag$, +'system' +); + +INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_by) VALUES ( +1443253234253454, +'test-workspace', +'f/dre/flow', +'{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {}, + "required": [], + "type": "object" +}', +$tag$ +{ + "modules": [ + { + "id": "a", + "value": { + "lock": "# py: 3.11\n", + "type": "rawscript", + "assets": [], + "content": "import f.dre.leaf_left\n\ndef main():\n pass", + "language": "python3", + "input_transforms": {} + }, + "summary": "leaf Left" + }, + { + "id": "b", + "value": { + "lock": "# py: 3.11\n", + "type": "rawscript", + "assets": [], + "content": "import f.dre.leaf_right\nimport f.dre.leaf_left\n\ndef main():\n pass", + "language": "python3", + "input_transforms": {} + }, + "summary": "leaf Left and Right" + }, + { + "id": "c", + "value": { + "lock": "# py: 3.11\n", + "type": "rawscript", + "assets": [], + "content": "import f.dre.leaf_right\n\ndef main():\n pass", + "language": "python3", + "input_transforms": {} + }, + "summary": "leaf RIght" + } + ] +}$tag$, +'system' +); + +-- APPS -- +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +'#requirements: +#bottle==0.13.2 +def main(): + pass +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/dre_app/leaf_left', 433400, 'python3', ''); +-- Padded Hex: 0000000000069CF8 + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +'#requirements: +#tiny==0.1.3 +def main(): + pass +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/dre_app/leaf_right', 433403, 'python3', ''); +-- Padded Hex: 0000000000069CFB + +INSERT INTO public.app(id, workspace_id, path, versions, policy) VALUES ( +2, +'test-workspace', +'f/dre_app/app', +'{0}', +'{}' +); + +INSERT INTO public.app_version(id, app_id, value, created_by) VALUES ( +0, +2, +$tag${"grid":[{"3":{"fixed":true,"x":0,"y":0,"fullHeight":false,"w":6,"h":2},"12":{"fixed":true,"x":0,"y":0,"fullHeight":false,"w":12,"h":2},"data":{"type":"containercomponent","configuration":{},"customCss":{"container":{"class":"!p-0","style":""}},"numberOfSubgrids":1,"id":"topbar"},"id":"topbar"},{"3":{"fixed":false,"x":0,"y":2,"fullHeight":false,"w":1,"h":1},"12":{"fixed":false,"x":0,"y":2,"fullHeight":false,"w":2,"h":1},"data":{"type":"buttoncomponent","configuration":{"label":{"type":"static","value":"A"},"color":{"type":"static","value":"blue"},"size":{"type":"static","value":"xs"},"fillContainer":{"type":"static","value":false},"disabled":{"type":"static","value":false},"beforeIcon":{"type":"static"},"afterIcon":{"type":"static"},"tooltip":{"type":"static","value":""},"triggerOnAppLoad":{"type":"static","value":false},"runInBackground":{"type":"static","value":false},"onSuccess":{"type":"oneOf","selected":"none","configuration":{"none":{},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"setTab":{"setTab":{"type":"static","value":[]}},"sendToast":{"message":{"type":"static","value":""}},"openModal":{"modalId":{"type":"static","value":""}},"closeModal":{"modalId":{"type":"static","value":""}},"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"clearFiles":{"id":{"type":"static","value":""}}}},"onError":{"type":"oneOf","selected":"errorOverlay","configuration":{"errorOverlay":{},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"setTab":{"setTab":{"type":"static","value":[]}},"sendErrorToast":{"message":{"type":"static","value":"An error occurred"},"appendError":{"type":"static","value":true}},"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}}}},"confirmationModal":{"type":"oneOf","selected":"none","configuration":{"none":{},"confirmationModal":{"title":{"type":"static","value":"Title"},"description":{"type":"static","value":"Are you sure?"},"confirmationText":{"type":"static","value":"Confirm"}}}}},"componentInput":{"type":"runnable","fieldType":"any","fields":{},"runnable":{"type":"runnableByName","name":"Inline Script","inlineScript":{"content":"import f.dre_app.leaf_left\n\ndef main():\n pass\n","language":"python3","schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"},"path":"f/dre_app/app/Inline_Script"}},"autoRefresh":false,"recomputeOnInputChanged":false},"customCss":{"button":{"style":"","class":""},"container":{"style":"","class":""}},"recomputeIds":[],"horizontalAlignment":"center","verticalAlignment":"center","id":"a"},"id":"a"},{"3":{"fixed":false,"x":1,"y":2,"fullHeight":false,"w":1,"h":1},"12":{"fixed":false,"x":2,"y":2,"fullHeight":false,"w":2,"h":1},"data":{"type":"buttoncomponent","configuration":{"label":{"type":"static","value":"B"},"color":{"type":"static","value":"blue"},"size":{"type":"static","value":"xs"},"fillContainer":{"type":"static","value":false},"disabled":{"type":"static","value":false},"beforeIcon":{"type":"static"},"afterIcon":{"type":"static"},"tooltip":{"type":"static","value":""},"triggerOnAppLoad":{"type":"static","value":false},"runInBackground":{"type":"static","value":false},"onSuccess":{"type":"oneOf","selected":"none","configuration":{"none":{},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"setTab":{"setTab":{"type":"static","value":[]}},"sendToast":{"message":{"type":"static","value":""}},"openModal":{"modalId":{"type":"static","value":""}},"closeModal":{"modalId":{"type":"static","value":""}},"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"clearFiles":{"id":{"type":"static","value":""}}}},"onError":{"type":"oneOf","selected":"errorOverlay","configuration":{"errorOverlay":{},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"setTab":{"setTab":{"type":"static","value":[]}},"sendErrorToast":{"message":{"type":"static","value":"An error occurred"},"appendError":{"type":"static","value":true}},"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}}}},"confirmationModal":{"type":"oneOf","selected":"none","configuration":{"none":{},"confirmationModal":{"title":{"type":"static","value":"Title"},"description":{"type":"static","value":"Are you sure?"},"confirmationText":{"type":"static","value":"Confirm"}}}}},"componentInput":{"type":"runnable","fieldType":"any","fields":{},"runnable":{"type":"runnableByName","name":"Inline Script","inlineScript":{"content":"import f.dre_app.leaf_left\nimport f.dre_app.leaf_right\n\ndef main():\n pass\n","language":"python3","schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"},"path":"f/dre_app/app/Inline_Script"}},"autoRefresh":false,"recomputeOnInputChanged":false},"customCss":{"button":{"style":"","class":""},"container":{"style":"","class":""}},"recomputeIds":[],"horizontalAlignment":"center","verticalAlignment":"center","id":"b"},"id":"b"},{"3":{"fixed":false,"x":2,"y":2,"fullHeight":false,"w":1,"h":1},"12":{"fixed":false,"x":4,"y":2,"fullHeight":false,"w":2,"h":1},"data":{"type":"buttoncomponent","configuration":{"label":{"type":"static","value":"C"},"color":{"type":"static","value":"blue"},"size":{"type":"static","value":"xs"},"fillContainer":{"type":"static","value":false},"disabled":{"type":"static","value":false},"beforeIcon":{"type":"static"},"afterIcon":{"type":"static"},"tooltip":{"type":"static","value":""},"triggerOnAppLoad":{"type":"static","value":false},"runInBackground":{"type":"static","value":false},"onSuccess":{"type":"oneOf","selected":"none","configuration":{"none":{},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"setTab":{"setTab":{"type":"static","value":[]}},"sendToast":{"message":{"type":"static","value":""}},"openModal":{"modalId":{"type":"static","value":""}},"closeModal":{"modalId":{"type":"static","value":""}},"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"clearFiles":{"id":{"type":"static","value":""}}}},"onError":{"type":"oneOf","selected":"errorOverlay","configuration":{"errorOverlay":{},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"setTab":{"setTab":{"type":"static","value":[]}},"sendErrorToast":{"message":{"type":"static","value":"An error occurred"},"appendError":{"type":"static","value":true}},"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}}}},"confirmationModal":{"type":"oneOf","selected":"none","configuration":{"none":{},"confirmationModal":{"title":{"type":"static","value":"Title"},"description":{"type":"static","value":"Are you sure?"},"confirmationText":{"type":"static","value":"Confirm"}}}}},"componentInput":{"type":"runnable","fieldType":"any","fields":{},"runnable":{"type":"runnableByName","name":"Inline Script","inlineScript":{"content":"import f.dre_app.leaf_right\n\ndef main():\n pass\n","language":"python3","schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"},"path":"f/dre_app/app/Inline_Script"}},"autoRefresh":false,"recomputeOnInputChanged":false},"customCss":{"button":{"style":"","class":""},"container":{"style":"","class":""}},"recomputeIds":[],"horizontalAlignment":"center","verticalAlignment":"center","id":"c"},"id":"c"}],"fullscreen":false,"unusedInlineScripts":[],"hiddenInlineScripts":[],"theme":{"type":"path","path":"f/app_themes/theme_0"},"subgrids":{"topbar-0":[{"3":{"fixed":false,"x":0,"y":0,"fullHeight":false,"w":6,"h":1},"12":{"fixed":false,"x":0,"y":0,"fullHeight":false,"w":6,"h":1},"data":{"type":"textcomponent","configuration":{"style":{"type":"static","value":"Body"},"copyButton":{"type":"static","value":false},"tooltip":{"type":"evalv2","value":"","fieldType":"text","expr":"`Author: ${ctx.author}`","connections":[{"componentId":"ctx","id":"author"}]},"disableNoText":{"type":"static","value":true,"fieldType":"boolean"}},"componentInput":{"type":"templatev2","fieldType":"template","eval":"${ctx.summary}","connections":[{"id":"summary","componentId":"ctx"}]},"customCss":{"text":{"class":"text-xl font-semibold whitespace-nowrap truncate","style":""},"container":{"class":"","style":""}},"horizontalAlignment":"left","verticalAlignment":"center","id":"title"},"id":"title"},{"3":{"fixed":false,"x":0,"y":1,"fullHeight":false,"w":3,"h":1},"12":{"fixed":false,"x":6,"y":0,"fullHeight":false,"w":6,"h":1},"data":{"type":"recomputeallcomponent","configuration":{"defaultRefreshInterval":{"type":"static","value":"0"}},"customCss":{"container":{"style":"","class":""}},"menuItems":[],"horizontalAlignment":"right","verticalAlignment":"center","id":"recomputeall"},"id":"recomputeall"}]},"hideLegacyTopBar":true,"mobileViewOnSmallerScreens":false}$tag$, +'system' +); + +-- SCRIPTS -- +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +'#requirements: +#bottle==0.13.2 +def main(): + pass +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/dre_script/leaf_left', 533400, 'python3', ''); +-- Padded Hex: 0000000000082398 + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +'#requirements: +#tiny==0.1.3 +def main(): + pass +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/dre_script/leaf_right', 533403, 'python3', ''); +-- Padded Hex: 000000000008239B + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +import f.dre_script.leaf_left +import f.dre_script.leaf_right + +def main(): + pass +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/dre_script/script', 533404, 'python3', ''); +-- Padded Hex: 000000000008239C + +-- Create dependency map +INSERT INTO dependency_map (workspace_id, imported_path, importer_kind, importer_path, importer_node_id) VALUES ('test-workspace', 'f/dre/leaf_left', 'flow', 'f/dre/flow', 'a'); +INSERT INTO dependency_map (workspace_id, imported_path, importer_kind, importer_path, importer_node_id) VALUES ('test-workspace', 'f/dre/leaf_left', 'flow', 'f/dre/flow', 'b'); +INSERT INTO dependency_map (workspace_id, imported_path, importer_kind, importer_path, importer_node_id) VALUES ('test-workspace', 'f/dre/leaf_right', 'flow', 'f/dre/flow', 'b'); +INSERT INTO dependency_map (workspace_id, imported_path, importer_kind, importer_path, importer_node_id) VALUES ('test-workspace', 'f/dre/leaf_right', 'flow', 'f/dre/flow', 'c'); + +INSERT INTO dependency_map (workspace_id, imported_path, importer_kind, importer_path, importer_node_id) VALUES ('test-workspace', 'f/dre_app/leaf_left', 'app', 'f/dre_app/app', 'a'); +INSERT INTO dependency_map (workspace_id, imported_path, importer_kind, importer_path, importer_node_id) VALUES ('test-workspace', 'f/dre_app/leaf_left', 'app', 'f/dre_app/app', 'b'); +INSERT INTO dependency_map (workspace_id, imported_path, importer_kind, importer_path, importer_node_id) VALUES ('test-workspace', 'f/dre_app/leaf_right', 'app', 'f/dre_app/app', 'b'); +INSERT INTO dependency_map (workspace_id, imported_path, importer_kind, importer_path, importer_node_id) VALUES ('test-workspace', 'f/dre_app/leaf_right', 'app', 'f/dre_app/app', 'c'); + +INSERT INTO dependency_map (workspace_id, imported_path, importer_kind, importer_path, importer_node_id) VALUES ('test-workspace', 'f/dre_script/leaf_left', 'script', 'f/dre_script/script', ''); +INSERT INTO dependency_map (workspace_id, imported_path, importer_kind, importer_path, importer_node_id) VALUES ('test-workspace', 'f/dre_script/leaf_right', 'script', 'f/dre_script/script', ''); diff --git a/backend/tests/fixtures/job_debouncing.sql b/backend/tests/fixtures/job_debouncing.sql new file mode 100644 index 0000000000..480d7c29a7 --- /dev/null +++ b/backend/tests/fixtures/job_debouncing.sql @@ -0,0 +1,171 @@ +-- SCRIPTS -- +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +'def main(x: str = "hey", b: int = 1): + pass +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/scripts/script_1', 533400, 'python3', ''); +-- Padded Hex: 0000000000082398 + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +'def main(): + pass +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/scripts/script_2', 533403, 'python3', ''); +-- Padded Hex: 000000000008239B + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +def main(): + pass +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/scripts/script_3', 533404, 'python3', ''); + + +-- Padded Hex: 000000000008239C +INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( +'test-workspace', +'', +'', +'f/flows/flow', +'{1443253234253454}', +'{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {}, + "required": [], + "type": "object" +}', +$tag$ +{ + "modules": [ + { + "id": "a", + "value": { + "type": "rawscript", + "assets": [], + "content": "def main(x: str, y: str):\n return x", + "language": "python3", + "debounce_delay_s": 2, + "input_transforms": { + "x": { + "type": "static", + "value": "" + }, + "y": { + "type": "static", + "value": "" + } + } + }, + "continue_on_error": false + } + ], + "debounce_delay_s": 2 +}$tag$, +'system' +); + +INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_by) +SELECT versions[1], workspace_id, path, schema, value, edited_by FROM flow WHERE path = 'f/flows/flow'; + +-- No top level debouncing +INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( +'test-workspace', +'', +'', +'f/flows/flow_full', +'{123}', +'{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {}, + "required": [], + "type": "object" +}', +$tag$ +{ + "modules": [ + { + "id": "a", + "value": { + "lock": "# py: 3.11\n", + "type": "rawscript", + "assets": [], + "content": "import time\n\ndef main(x: str, y: str):\n time.sleep(30)\n\n return x", + "language": "python3", + "concurrent_limit": 1, + "input_transforms": { + "x": { + "type": "static", + "value": "" + }, + "y": { + "type": "static", + "value": "" + } + }, + "concurrency_time_window_s": 5 + }, + "continue_on_error": false + }, + { + "id": "b", + "value": { + "type": "whileloopflow", + "modules": [ + { + "id": "c", + "value": { + "lock": "# py: 3.11\n", + "type": "rawscript", + "assets": [], + "content": "# import wmill\n\n\ndef main(x: str):\n return x", + "language": "python3", + "input_transforms": { + "x": { + "type": "static", + "value": "" + } + } + } + }, + { + "id": "d", + "value": { + "type": "rawscript", + "assets": [], + "content": "# import wmill\n\n\ndef main(x: str):\n return x", + "language": "python3", + "input_transforms": { + "x": { + "type": "static", + "value": "" + } + } + } + } + ], + "skip_failures": false + } + } + ] +}$tag$, +'system' +); + + +INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_by) +SELECT versions[1], workspace_id, path, schema, value, edited_by FROM flow WHERE path = 'f/flows/flow_full'; diff --git a/backend/tests/fixtures/relative_python.sql b/backend/tests/fixtures/relative_python.sql index 05e8453ccc..7a70e06d20 100644 --- a/backend/tests/fixtures/relative_python.sql +++ b/backend/tests/fixtures/relative_python.sql @@ -22,7 +22,6 @@ def main(): '', 'f/system_relative/different_folder_script', 12347, 'python3', ''); - INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( 'test-workspace', 'test-user', @@ -38,4 +37,4 @@ def main(): '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', '', '', -'f/system_relative/nested_script', 12348, 'python3', ''); \ No newline at end of file +'f/system_relative/nested_script', 12348, 'python3', ''); diff --git a/backend/tests/job_payload.rs b/backend/tests/job_payload.rs index d01064d4bc..22e713329b 100644 --- a/backend/tests/job_payload.rs +++ b/backend/tests/job_payload.rs @@ -1,19 +1,19 @@ mod common; - mod job_payload { use serde_json::json; + use sqlx::{Pool, Postgres}; use std::sync::Arc; use tokio::sync::RwLock; - use sqlx::{Pool, Postgres}; - use windmill_common::scripts::{ScriptHash, ScriptLang}; - use windmill_common::jobs::JobPayload; - use windmill_common::flows::{FlowValue, FlowModule, FlowModuleValue}; use windmill_common::flow_status::RestartedFrom; + use windmill_common::flows::{FlowModule, FlowModuleValue, FlowValue}; + use windmill_common::jobs::JobPayload; + use windmill_common::scripts::{ScriptHash, ScriptLang}; + + use crate::common::*; use windmill_common::worker::{ MIN_VERSION_IS_AT_LEAST_1_427, MIN_VERSION_IS_AT_LEAST_1_432, MIN_VERSION_IS_AT_LEAST_1_440, }; - use crate::common::*; pub async fn initialize_tracing() { use std::sync::Once; @@ -41,11 +41,9 @@ mod job_payload { ]; } - #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base", "hello"))] async fn test_script_hash_payload(db: Pool) -> anyhow::Result<()> { - initialize_tracing().await; let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); @@ -57,6 +55,8 @@ mod job_payload { custom_concurrency_key: None, concurrent_limit: None, concurrency_time_window_s: None, + custom_debounce_key: None, + debounce_delay_s: None, cache_ttl: None, dedicated_worker: None, language: ScriptLang::Deno, @@ -64,7 +64,7 @@ mod job_payload { apply_preprocessor: false, }) .arg("world", json!("foo")) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -94,8 +94,10 @@ mod job_payload { language: ScriptLang::Deno, priority: None, apply_preprocessor: true, + custom_debounce_key: None, + debounce_delay_s: None, }) - .run_until_complete_with(db, port, |id| async move { + .run_until_complete_with(db, false, port, |id| async move { let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) .fetch_one(db) .await @@ -130,7 +132,7 @@ mod job_payload { dedicated_worker: None, version: 1443253234253454, }) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -173,7 +175,7 @@ mod job_payload { path: "f/system/hello/test-0".into(), }) .arg("world", json!("foo")) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -193,7 +195,7 @@ mod job_payload { path: "f/system/hello/test-0".into(), }) .arg("hello", json!("You know nothing Jean Neige")) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -220,7 +222,7 @@ mod job_payload { dedicated_worker: None, version: 1443253234253454, }) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -241,7 +243,7 @@ mod job_payload { path: "f/system/hello_with_nodes_flow/forloop-0".into(), }) .arg("iter", json!({ "value": "tests", "index": 0 })) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -263,7 +265,7 @@ mod job_payload { language: ScriptLang::Deno, dedicated_worker: None, }) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -307,7 +309,7 @@ mod job_payload { dedicated_worker: None, version: 1443253234253454, }) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -356,7 +358,7 @@ mod job_payload { .unwrap(), }) .arg("skip_flow_update", json!(true)) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -370,7 +372,7 @@ mod job_payload { restarted_from: None, }) .arg("world", json!("Jean Neige")) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -398,7 +400,7 @@ mod job_payload { .into(), language: ScriptLang::Deno, }) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -426,7 +428,7 @@ mod job_payload { apply_preprocessor: false, version: 1443253234253454, }) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -448,7 +450,7 @@ mod job_payload { dedicated_worker: None, version: 1443253234253454, }) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -474,7 +476,7 @@ mod job_payload { apply_preprocessor: true, version: 1443253234253456, }) - .run_until_complete_with(db, port, |id| async move { + .run_until_complete_with(db, false, port, |id| async move { let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) .fetch_one(db) .await @@ -520,7 +522,7 @@ mod job_payload { dedicated_worker: None, version: 1443253234253456, }) - .run_until_complete(db, port) + .run_until_complete(db, false, port) .await .json_result() .unwrap(); @@ -543,7 +545,7 @@ mod job_payload { apply_preprocessor: true, version: 1443253234253454, }) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .id; @@ -553,7 +555,7 @@ mod job_payload { branch_or_iteration_n: None, }) .arg("iter", json!({ "value": "tests", "index": 0 })) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -575,7 +577,7 @@ mod job_payload { dedicated_worker: None, version: 1443253234253454, }) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -621,7 +623,7 @@ mod job_payload { restarted_from: None, }) .arg("world", json!("Jean Neige")) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -701,7 +703,7 @@ mod job_payload { restarted_from, }) .arg("world", arg) - .run_until_complete(db, port) + .run_until_complete(db, false, port) .await; assert_eq!(job.json_result().unwrap(), result); diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index c13c5c7876..afa662f582 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -1,8 +1,8 @@ mod common; use crate::common::*; -use sqlx::Pool; use sqlx::postgres::Postgres; -use windmill_common::scripts::{ ScriptLang}; +use sqlx::Pool; +use windmill_common::scripts::ScriptLang; #[cfg(feature = "python")] #[sqlx::test(fixtures("base", "lockfile_python"))] @@ -162,7 +162,6 @@ use windmill_common::jobs::RawCode; #[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_python_job(db: Pool) -> anyhow::Result<()> { - initialize_tracing().await; let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); @@ -182,11 +181,13 @@ def main(): custom_concurrency_key: None, concurrent_limit: None, concurrency_time_window_s: None, + custom_debounce_key: None, + debounce_delay_s: None, cache_ttl: None, dedicated_worker: None, }); - let result = run_job_in_new_worker_until_complete(&db, job, port) + let result = run_job_in_new_worker_until_complete(&db, false, job, port) .await .json_result() .unwrap(); @@ -211,7 +212,6 @@ async fn test_python_global_site_packages(db: Pool) -> anyhow::Result< // 3.12 { - let content = r#"# py: ==3.12 #requirements: # @@ -232,11 +232,13 @@ def main(): custom_concurrency_key: None, concurrent_limit: None, concurrency_time_window_s: None, + custom_debounce_key: None, + debounce_delay_s: None, cache_ttl: None, dedicated_worker: None, }); - let result = run_job_in_new_worker_until_complete(&db, job, port) + let result = run_job_in_new_worker_until_complete(&db, false, job, port) .await .json_result() .unwrap(); @@ -266,11 +268,13 @@ def main(): custom_concurrency_key: None, concurrent_limit: None, concurrency_time_window_s: None, + custom_debounce_key: None, + debounce_delay_s: None, cache_ttl: None, dedicated_worker: None, }); - let result = run_job_in_new_worker_until_complete(&db, job, port) + let result = run_job_in_new_worker_until_complete(&db, false, job, port) .await .json_result() .unwrap(); @@ -305,11 +309,13 @@ def main(): custom_concurrency_key: None, concurrent_limit: None, concurrency_time_window_s: None, + custom_debounce_key: None, + debounce_delay_s: None, cache_ttl: None, dedicated_worker: None, }); - let result = run_job_in_new_worker_until_complete(&db, job, port) + let result = run_job_in_new_worker_until_complete(&db, false, job, port) .await .json_result() .unwrap(); @@ -342,11 +348,13 @@ def main(): custom_concurrency_key: None, concurrent_limit: None, concurrency_time_window_s: None, + custom_debounce_key: None, + debounce_delay_s: None, cache_ttl: None, dedicated_worker: None, }); - let result = run_job_in_new_worker_until_complete(&db, job, port) + let result = run_job_in_new_worker_until_complete(&db, false, job, port) .await .json_result() .unwrap(); @@ -355,7 +363,6 @@ def main(): Ok(()) } - #[cfg(feature = "python")] #[sqlx::test(fixtures("base", "relative_python"))] async fn test_relative_imports_python(db: Pool) -> anyhow::Result<()> { @@ -391,4 +398,3 @@ def main(): run_preview_relative_imports(&db, content, ScriptLang::Python3).await?; Ok(()) } - diff --git a/backend/tests/relative_imports.rs b/backend/tests/relative_imports.rs new file mode 100644 index 0000000000..2de18732b4 --- /dev/null +++ b/backend/tests/relative_imports.rs @@ -0,0 +1,3273 @@ +// TODO: move all related logic here (if anything left anywhere in codebase) +mod common; +use windmill_api_client::types::NewScript; + +fn quick_ns( + content: &str, + language: windmill_api_client::types::ScriptLang, + path: &str, + lock: Option, + parent_hash: Option, +) -> NewScript { + NewScript { + content: content.into(), + language, + lock, + parent_hash, + path: path.into(), + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + description: "".to_string(), + draft_only: None, + envs: vec![], + is_template: None, + kind: None, + summary: "".to_string(), + tag: None, + schema: std::collections::HashMap::new(), + ws_error_handler_muted: Some(false), + priority: None, + delete_after_use: None, + timeout: None, + restart_unless_cancelled: None, + deployment_message: None, + concurrency_key: None, + visible_to_runner_only: None, + no_main_func: None, + codebase: None, + has_preprocessor: None, + on_behalf_of_email: None, + assets: vec![], + } +} + +mod dependency_map { + use super::quick_ns; + use sqlx::{Pool, Postgres}; + use tokio_stream::StreamExt; + + use crate::common::{in_test_worker, init_client, listen_for_completed_jobs, ApiServer}; + + async fn init(db: Pool) -> (windmill_api_client::Client, u16, ApiServer) { + init_client(db).await + } + + async fn _clear_dmap(db: &Pool) { + sqlx::query!("DELETE FROM dependency_map WHERE workspace_id = 'test-workspace'") + .execute(db) + .await + .unwrap(); + } + + /// Corrects map according to provided replacements. + /// Only changes importer_path and/or id + /// Does not affect imported_path nor kind! + fn corrected_dmap(replacements: Vec<(&str, &str)>) -> Vec<(String, String, String, String)> { + CORRECT_DMAP + .clone() + .into_iter() + .map(|e| { + let mut r = ( + e.0.to_owned(), + e.1.to_owned(), + e.2.to_owned(), + e.3.to_owned(), + ); + for (from, to) in &replacements { + r = ( + r.0.replace(from, to), + r.1, // Kind should be immutable + r.2, // Imported path should be immutable + // We do not modify script contents in test, so we can assume scripts always import the same path + // Modification of kind or imported path considered to be incorrect. + r.3.replace(from, to), + ); + } + r + }) + .collect() + } + + async fn assert_dmap( + db: &Pool, + importer: Option, + expected: Vec<( + impl Into, + impl Into, + impl Into, + impl Into, + )>, + ) { + let dmap = sqlx::query_as::<_, (String, String, String, String)>( + "SELECT importer_path, importer_kind::text, imported_path, importer_node_id FROM dependency_map WHERE workspace_id = 'test-workspace' AND ($1::text IS NULL OR importer_path = $1::text)", + ) + .bind(importer) + .fetch_all(db) + .await + .unwrap(); + + assert_eq!( + dmap, + expected + .into_iter() + .map(|(f, s, t, fo)| (f.into(), s.into(), t.into(), fo.into())) + .collect::>() + ); + } + + lazy_static::lazy_static! { + pub static ref CORRECT_DMAP: Vec<(&'static str, &'static str, &'static str, &'static str)> = vec![ + ("f/rel/branch", "script", "f/rel/leaf_1", ""), + ("f/rel/root_script", "script", "f/rel/branch", ""), + ("f/rel/root_script", "script", "f/rel/leaf_1", ""), + ("f/rel/root_script", "script", "f/rel/leaf_2", ""), + ("f/rel/root_app", "app", "f/rel/leaf_2", "dontpressmeplz"), + ("f/rel/root_flow", "flow", "f/rel/leaf_2", "failure"), + ("f/rel/root_flow", "flow", "f/rel/branch", "nstep1"), + ("f/rel/root_flow", "flow", "f/rel/leaf_1", "nstep1"), + ("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep1"), + ("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep2_2"), + ("f/rel/root_flow", "flow", "f/rel/branch", "nstep4_1"), + ("f/rel/root_flow", "flow", "f/rel/branch", "nstep5_1"), + ("f/rel/root_flow", "flow", "f/rel/leaf_1", "nstep5_1"), + ("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep5_1"), + ("f/rel/root_flow", "flow", "f/rel/branch", "preprocessor"), + ("f/rel/root_flow", "flow", "f/rel/leaf_1", "preprocessor"), + ("f/rel/root_flow", "flow", "f/rel/leaf_2", "preprocessor"), + ("f/rel/root_app", "app", "f/rel/branch", "pressmeplz"), + ("f/rel/root_app", "app", "f/rel/leaf_1", "pressmeplz"), + ("f/rel/root_app", "app", "f/rel/leaf_2", "pressmeplz"), + ("f/rel/root_flow", "flow", "f/rel/leaf_2", "qtool1"), + ("f/rel/root_app", "app", "f/rel/branch", "youcanpressme")]; + } + + // TODO: + // Test that checks that we can run rebuild_dmap multiple times in tests. + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rebuild_correctness(db: Pool) -> anyhow::Result<()> { + let (client, _port, _s) = init(db.clone()).await; + assert_dmap(&db, None, CORRECT_DMAP.clone()).await; + // rebuild map + assert!(super::common::rebuild_dmap(&client).await); + assert_dmap(&db, None, CORRECT_DMAP.clone()).await; + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rebuild_lock(db: Pool) -> anyhow::Result<()> { + let (client, _port, _s) = init(db.clone()).await; + + // Spawn first rebuild + let handle = { + let client = client.clone(); + tokio::spawn(async move { super::common::rebuild_dmap(&client).await }) + }; + + // Immidiately spawn another + let res = client + .client() + .post(format!( + "{}/w/test-workspace/workspaces/rebuild_dependency_map", + client.baseurl() + )) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + + // Should tell us there is already rebuilt in progress + // Or if it is too fast we will be able to trigger it second time + assert!(&res == "There is already one task pending, try again later." || &res == "Success"); + + assert!(handle.await.unwrap()); + Ok(()) + } + + // If you deploy from cli and you use raw requirements you don't want the script be included in dmap + // Otherwise script will be overwritten once any relative import is updated + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_with_requirements_txt(db: Pool) -> anyhow::Result<()> { + let (client, _port, _s) = init(db.clone()).await; + + client + .create_script( + "test-workspace", + &quick_ns( + " +from f.rel.branch import main as br; +from f.rel.leaf_1 import main as lf_1; +from f.rel.leaf_2 import main as lf_2; + +def main(): + return [br(), lf_1(), lf_2]; + ", + windmill_api_client::types::ScriptLang::Python3, + "f/rel/root_script", + Some("# from requirements.txt".to_string()), + Some("000000000005165B".into()), + ), + ) + .await + .unwrap(); + + assert_dmap( + &db, + Some("f/rel/root_script".into()), + vec![ + ("f/rel/root_script", "script", "f/rel/branch", ""), + ("f/rel/root_script", "script", "f/rel/leaf_1", ""), + ("f/rel/root_script", "script", "f/rel/leaf_2", ""), + ], + ) + .await; + + tokio::time::sleep(std::time::Duration::from_secs(13)).await; + + assert_dmap( + &db, + Some("f/rel/root_script".into()), + Vec::<(String, String, String, String)>::new(), + ) + .await; + + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_without_requirements_txt( + db: Pool, + ) -> anyhow::Result<()> { + let (client, _port, _s) = init(db.clone()).await; + + client + .create_script( + "test-workspace", + &quick_ns( + " +from f.rel.branch import main as br; +from f.rel.leaf_1 import main as lf_1; +from f.rel.leaf_2 import main as lf_2; + +def main(): + return [br(), lf_1(), lf_2]; + ", + windmill_api_client::types::ScriptLang::Python3, + "f/rel/root_script", + // We still want to pass lock to it. + Some("# py311".to_string()), + Some("000000000005165B".into()), + ), + ) + .await + .unwrap(); + assert_dmap(&db, None, CORRECT_DMAP.clone()).await; + // tokio::time::sleep(std::time::Duration::from_secs(13)).await; + assert_dmap(&db, None, CORRECT_DMAP.clone()).await; + Ok(()) + } + // Consider simple one. Only referenced directly. No deep connections + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rename_leaf_2(db: Pool) -> anyhow::Result<()> { + let (client, port, _s) = init(db.clone()).await; + client + .create_script( + "test-workspace", + &quick_ns( + " +def main(): + return 'leaf3'; + ", + windmill_api_client::types::ScriptLang::Python3, + "f/rel/leaf_2_renamed", + None, + Some("0000000000051659".into()), + ), + ) + .await + .unwrap(); + + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker(&db, completed.next(), port).await; + + // Changing leafs should not change dependency map + assert_dmap(&db, None, CORRECT_DMAP.clone()).await; + Ok(()) + } + + // Consider hard one. Referenced deeply and exists in double references. + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rename_leaf_1(db: Pool) -> anyhow::Result<()> { + let (client, port, _s) = init(db.clone()).await; + client + .create_script( + "test-workspace", + &quick_ns( + " +def main(): + return 'leaf1'; + ", + windmill_api_client::types::ScriptLang::Python3, + "f/rel/leaf_1_renamed", + None, + Some("0000000000051658".into()), + ), + ) + .await + .unwrap(); + + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker(&db, completed.next(), port).await; + + // Changing leafs should not change dependency map + assert_dmap(&db, None, CORRECT_DMAP.clone()).await; + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rename_branch(db: Pool) -> anyhow::Result<()> { + let (client, port, _s) = init(db.clone()).await; + client + .create_script( + "test-workspace", + &quick_ns( + " +from f.rel.leaf_1 import main as lf_1; + +def main(): + return lf_1(); + ", + windmill_api_client::types::ScriptLang::Python3, + "f/rel/branch_renamed", + None, + Some("000000000005165A".into()), + ), + ) + .await + .unwrap(); + + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker(&db, completed.next(), port).await; + + // Changing branches SHOULD change dependency map + // Though it should only change branch item in dmap when it is importer. + // All entries when branch is imported should not change. + let mut corrected_dmap = CORRECT_DMAP.clone(); + // Corresponds to importer path of branch entry + corrected_dmap[0].0 = "f/rel/branch_renamed"; + assert_dmap(&db, None, corrected_dmap).await; + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rename_primary_script(db: Pool) -> anyhow::Result<()> { + let (client, port, _s) = init(db.clone()).await; + + client + .create_script( + "test-workspace", + &quick_ns( + " +from f.rel.branch import main as br; +from f.rel.leaf_1 import main as lf_1; +from f.rel.leaf_2 import main as lf_2; + +def main(): + return [br(), lf_1(), lf_2]; + ", + windmill_api_client::types::ScriptLang::Python3, + "f/rel/root_script_renamed", + None, + Some("000000000005165B".into()), + ), + ) + .await + .unwrap(); + + let corrected_dmap = corrected_dmap(vec![("root_script", "root_script_renamed")]); + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker(&db, completed.next(), port).await; + assert_dmap(&db, None, corrected_dmap.clone()).await; + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rename_primary_flow(db: Pool) -> anyhow::Result<()> { + use windmill_common::{cache::flow::fetch_version, flows::NewFlow, worker::to_raw_value}; + + let (client, port, _s) = init(db.clone()).await; + let flow = fetch_version(&db, 1443253234253454).await.unwrap(); + let res = client + .client() + .post(format!( + "{}/w/test-workspace/flows/update/{}", + client.baseurl(), + "f/rel/root_flow" // encode_path() + )) + .json(&NewFlow { + path: "f/rel/root_flow_renamed".into(), + summary: "".into(), + description: None, + value: to_raw_value(&serde_json::from_str::( + &serde_json::to_string(flow.value()) + .unwrap() + .replace("nstep1", "Foxes") + .replace("nstep2_2", "like") + .replace("nstep_4_1", "Emeralds"), + ) + .unwrap()), + schema: None, + draft_only: None, + tag: None, + dedicated_worker: None, + timeout: None, + deployment_message: None, + visible_to_runner_only: None, + on_behalf_of_email: None, + ws_error_handler_muted: None + }) + .send() + .await + .unwrap(); + + assert_eq!(res.text().await.unwrap(), "f/rel/root_flow_renamed"); + + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker(&db, completed.next(), port).await; + + assert_dmap( + &db, + None, + corrected_dmap(vec![ + ("f/rel/root_flow", "f/rel/root_flow_renamed"), + ("nstep1", "Foxes"), + ("nstep2_2", "like"), + ("nstep_4_1", "Emeralds"), + ]), + ) + .await; + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rename_primary_app(db: Pool) -> anyhow::Result<()> { + let (client, port, _s) = init(db.clone()).await; + + let app_value: String = + sqlx::query_scalar!("SELECT value::text FROM app_version WHERE id = 0 AND app_id = 2") + .fetch_one(&db) + .await + .unwrap() + .unwrap(); + + // TODO: There is: + // 1. update app + // 2. create app + // 3. update app raw + // Ideally all of them should be handled + let res = client + .client() + .post(format!( + "{}/w/test-workspace/apps/update/{}", + client.baseurl(), + "f/rel/root_app" // encode_path() + )) + .json(&windmill_api::EditApp { + path: Some("f/rel/root_app_renamed".into()), + summary: None, + value: serde_json::from_str( + &app_value + .replace("dontpressmeplz", "Apps") + .replace("youcanpressme", "Work"), + ) + .unwrap(), + policy: None, + deployment_message: None, + custom_path: None, + }) + .send() + .await + .unwrap(); + + assert_eq!( + res.text().await.unwrap(), + "app f/rel/root_app updated (npath: \"f/rel/root_app_renamed\")" + ); + + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker(&db, completed.next(), port).await; + + assert_dmap( + &db, + None, + corrected_dmap(vec![ + ("f/rel/root_app", "f/rel/root_app_renamed"), + ("dontpressmeplz", "Apps"), + ("youcanpressme", "Work"), + ]), + ) + .await; + Ok(()) + } +} + +#[cfg(feature = "test_job_debouncing")] +mod dependency_job_debouncing { + async fn trigger_djob_for( + client: &windmill_api_client::Client, + path: &str, + parent_hash: &str, + content: Option, + ) { + use super::quick_ns; + use windmill_api_client::types::ScriptLang; + client + .create_script( + "test-workspace", + &quick_ns( + &content.unwrap_or( + " +def main(): + pass + " + .into(), + ), + ScriptLang::Python3, + path, + None, + Some(parent_hash.into()), + ), + ) + .await + .unwrap(); + } + // TODO: test workspaces specific things, + + /// # Double referenced even + /// It follows this topology: + /// + /// ┌─FLOW──────────┐ + /// │┌───┐┌───┐┌───┐│ + /// ││ A ││ B ││ C ││ + /// │└─▲─┘▲───▲└─▲─┘│ + /// └──┼──┼───┼──┼──┘ + /// ┌┴──┴┐ ┌┴──┴┐ + /// │L_LF│ │R_LF│ + /// └────┘ └────┘ + /// + /// p.s: "LF" stands for "Leaf", "L" - "Left", "R" - "Right" + mod flows { + use crate::common::{in_test_worker, init_client, listen_for_completed_jobs}; + use crate::dependency_job_debouncing::trigger_djob_for; + use std::time::Duration; + use tokio::time::sleep; + use tokio_stream::StreamExt; + + /// 1. LLF and RLF create two djobs for flow at the same and fall into single debounce + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "djob_debouncing"))] + async fn test_1(db: sqlx::Pool) -> anyhow::Result<()> { + // This tests if debouncing and consolidation works. + // Also makes sures that dependency job does not create new flow version + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = true; + } + + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + // Verify locks are empty + { + assert_eq!( + sqlx::query_scalar!("SELECT jsonb_array_elements(value->'modules')->'value'->>'lock' AS lock FROM flow") + .fetch_all(&db) + .await + .unwrap(), + vec![ + Some("# py: 3.11\n".into()), + Some("# py: 3.11\n".into()), + Some("# py: 3.11\n".into()) + ] + ); + } + + // Trigger both at the same time. + { + trigger_djob_for( + &client, + "f/dre/leaf_left", + "0000000000051658", + Some("#requirements:\n#bottle==0.13.2\ndef main():\npass".into()), + ) + .await; + + trigger_djob_for( + &client, + "f/dre/leaf_right", + "000000000005165B", + Some("#requirements:\n#tiny==0.1.3\ndef main():\npass".into()), + ) + .await; + } + + in_test_worker( + &db, + async { + assert_eq!( + &sqlx::query_scalar!( + "SELECT runnable_path FROM v2_job WHERE id = $1", + completed.next().await.unwrap() + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + "f/dre/leaf_left" + ); + + assert_eq!( + &sqlx::query_scalar!( + "SELECT runnable_path FROM v2_job WHERE id = $1", + completed.next().await.unwrap() + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + "f/dre/leaf_right" + ); + + // Let jobs propagate + sleep(Duration::from_secs(2)).await; + + // Verify there is only one queued job that is scheduled for atleast 3s ahead. + { + let q = sqlx::query_scalar!( + "SELECT (scheduled_for - created_at) FROM v2_job_queue" + ) + .fetch_all(&db) + .await + .unwrap(); + + assert_eq!(1, q.len()); + assert!(dbg!(q[0].unwrap().microseconds) > 1_000_000 /* 1 second */); + } + + // Verify debounce_stale_data and debounce_key + { + let q = sqlx::query!( + "SELECT + dsd.to_relock, + dk.key + FROM debounce_key dk + JOIN debounce_stale_data dsd ON dk.job_id = dsd.job_id" + ) + .fetch_all(&db) + .await + .unwrap(); + + // Should be single entry + assert!(q.len() == 1); + + // This verifies that all nodes_to_relock are consolidated correctly + // AND there is no doublicats + assert_eq!( + q[0].to_relock.clone().unwrap(), + vec!["a".to_owned(), "b".to_owned(), "c".to_owned()] + ); + + // Should be workspace specific and these specific tests cover only dependency job debouncing + assert_eq!( + q[0].key.clone(), + "test-workspace:f/dre/flow:dependency".to_owned(), + ); + } + + assert_eq!( + &sqlx::query_scalar!( + "SELECT runnable_path FROM v2_job WHERE id = $1", + completed.next().await.unwrap() + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + "f/dre/flow" + ); + }, + port, + ) + .await; + + // Verify latest flow.version property + { + // Latest flow version should not be initial one + assert_eq!( + 1, // Automatically assigned + dbg!(sqlx::query_scalar!( + "SELECT versions[2] FROM flow WHERE path = 'f/dre/flow'" + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap()) + ); + + // Only second element should be our initial version + assert_eq!( + 1443253234253454, // < Predefined in fixture + dbg!(sqlx::query_scalar!( + "SELECT versions[1] FROM flow WHERE path = 'f/dre/flow'" + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap()) + ); + } + + // Verify that there is only two versions of flow in global flow_version + { + assert_eq!( + 2, + sqlx::query_scalar!( + "SELECT COUNT(*) FROM flow_version WHERE path = 'f/dre/flow'" + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap() + ); + } + + // Verify locks + { + assert_eq!( + sqlx::query_scalar!("SELECT jsonb_array_elements(value->'modules')->'value'->>'lock' AS lock FROM flow") + .fetch_all(&db) + .await + .unwrap(), + vec![ + Some("# py: 3.11\nbottle==0.13.2".into()), + Some("# py: 3.11\nbottle==0.13.2\ntiny==0.1.3".into()), + Some("# py: 3.11\ntiny==0.1.3".into()) + ] + ); + } + + // TODO: + // tracing_assertions::assert_has_events!([info("This is supposed to be called")]); + // 2025-10-06T14:31:10.832469Z WARN windmill-worker/src/worker.rs:1593: pull took more than 0.1s (0.222477345) this is a sign that the database is undersized for this load. empty: true, err: true worker=wk-default-nixos-EzDEL hostname=nixos + + // Verify cleanup + { + assert_eq!( + 0, + sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") + .fetch_one(&db) + .await + .unwrap() + .unwrap() + ); + assert_eq!( + 0, + sqlx::query_scalar!("SELECT COUNT(*) from debounce_stale_data") + .fetch_one(&db) + .await + .unwrap() + .unwrap() + ); + } + + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "djob_debouncing"))] + async fn test_left(db: sqlx::Pool) -> anyhow::Result<()> { + use crate::common::RunJob; + + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = true; + } + // TODO: We don't care about timer. If there is no timer, it will be set automatically for djobs?? + let (_client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + // Trigger both at the same time. + { + let mut args = std::collections::HashMap::new(); + args.insert( + "dbg_djob_sleep".to_owned(), + // Execution should take this seconds + windmill_common::worker::to_raw_value(&20), + ); + + args.insert( + "triggered_by_relative_import".to_owned(), + // Execution should take this seconds + windmill_common::worker::to_raw_value(&()), + ); + + let (_flow_id, new_tx) = windmill_queue::push( + &db, + windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), + "test-workspace", + windmill_common::jobs::JobPayload::FlowDependencies { + path: "f/dre/flow".to_owned(), + dedicated_worker: None, + version: 1443253234253454, + }, + windmill_queue::PushArgs { args: &args, extra: None }, + "admin", + "admin@windmill.dev", + "admin".to_owned(), + Some("trigger.dependents.to.recompute.dependencies"), + // Debounce period + Some(chrono::Utc::now() + chrono::Duration::seconds(5)), + None, + None, + None, + None, + None, + false, + false, + None, + true, + Some("dependency".into()), + None, + None, + None, + None, + false, + None, + None, + ) + .await + .unwrap(); + new_tx.commit().await.unwrap(); + + // let handle = { + // // let mut completed = listen_for_completed_jobs(&db).await; + // let db2 = db.clone(); + // // let uuid = flow_id.clone(); + // tokio::spawn(async move { + // in_test_worker( + // &db2, + // tokio::time::sleep(tokio::time::Duration::from_secs(60)), + // // completed.find(&uuid), + // port, + // ) + // .await; + // }) + // }; + + let db2 = db.clone(); + in_test_worker( + &db2, + async { + // This job should execute and then try to start another job that will get debounced. + RunJob::from(windmill_common::jobs::JobPayload::Dependencies { + path: "f/dre/leaf_right".to_owned(), + hash: 333403.into(), + language: windmill_common::scripts::ScriptLang::Python3, + dedicated_worker: None, + }) + // .arg("dbg_djob_sleep", serde_json::json!(10)) + .run_until_complete(&db, false, port) + .await; + + RunJob::from(windmill_common::jobs::JobPayload::Dependencies { + path: "f/dre/leaf_left".to_owned(), + hash: 333400.into(), + language: windmill_common::scripts::ScriptLang::Python3, + dedicated_worker: None, + }) + // So set it to this long + .arg("dbg_djob_sleep", serde_json::json!(10)) + .run_until_complete(&db, false, port) + .await; + + completed.next().await; // leaf_right + completed.next().await; // leaf_left + completed.next().await; // importer + completed.next().await; // importer + }, + port, + ) + .await; + } + + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + 0 + ); + + let r = sqlx::query_scalar!("SELECT runnable_id FROM v2_job ORDER BY created_at DESC") + .fetch_all(&db) + .await + .unwrap(); + + assert_eq!(r.len(), 4); + assert!(r.contains(&Some(1))); + assert!(r.contains(&Some(333400))); + assert!(r.contains(&Some(333403))); + assert!(r.contains(&Some(1443253234253454))); + + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "djob_debouncing"))] + async fn test_2(db: sqlx::Pool) -> anyhow::Result<()> { + let (_client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = true; + } + + { + let mut mvsd = windmill_common::worker::MIN_VERSION_IS_AT_LEAST_1_440 + .write() + .await; + *mvsd = true; + } + + // Function to create a dependency job + let create_dependency_job = + |delay, + nodes_to_relock, + db: sqlx::Pool, + version, + debounce_job_id_o| async move { + let mut args = std::collections::HashMap::new(); + args.insert( + "dbg_sleep_between_pull_and_debounce_key_removal".to_owned(), + windmill_common::worker::to_raw_value(&delay), + ); + + args.insert( + "nodes_to_relock".to_owned(), + windmill_common::worker::to_raw_value(&nodes_to_relock), + ); + + args.insert( + "triggered_by_relative_import".to_string(), + windmill_common::worker::to_raw_value(&()), + ); + + args.insert( + "dbg_create_job_for_unexistant_flow_version".to_string(), + windmill_common::worker::to_raw_value(&()), + ); + + let (job_uuid, new_tx) = windmill_queue::push( + &db, + windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), + "test-workspace", + windmill_common::jobs::JobPayload::FlowDependencies { + path: "f/dre/flow".to_owned(), + dedicated_worker: None, + version, + }, + windmill_queue::PushArgs { args: &args, extra: None }, + "admin", + "admin@windmill.dev", + "admin".to_owned(), + Some("trigger.dependents.to.recompute.dependencies"), + Some(chrono::Utc::now()), // Schedule immediately + None, + None, + None, + None, + None, + false, + false, + None, + true, + Some("dependency".into()), + None, + None, + None, + None, + false, + None, + debounce_job_id_o, + ) + .await + .unwrap(); + + new_tx.commit().await.unwrap(); + job_uuid + }; + + // Push the first dependency job + let job1 = + create_dependency_job(2, vec!["a", "b"], db.clone(), 1443253234253454, None).await; + + let db2 = db.clone(); + in_test_worker( + &db2, + async { + // Small delay to ensure the job is marked as running + tokio::time::sleep(tokio::time::Duration::from_millis(200)).await; + + // Now is the time when the job is pulled, but debounce_key is not yet cleared. + { + assert!(sqlx::query_scalar!( + "SELECT running FROM v2_job_queue WHERE id = $1", + job1 + ) + .fetch_one(&db) + .await + .unwrap()); + + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM debounce_key") + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + 1 + ); + + dbg!(sqlx::query_scalar!("SELECT kind::text FROM v2_job") + .fetch_one(&db) + .await + .unwrap() + .unwrap()); + } + + let job2 = + create_dependency_job(0, vec!["b", "c"], db.clone(), 1, Some(job1)).await; + + // Process the first job completion, and the second job should also get debounced by this one + completed.next().await; + + // Verify that both jobs were created and processed + assert_eq!(job1, job2, "Second job should be debounced"); + }, + port, + ) + .await; + + assert_eq!( + vec![1443253234253454, 1], + sqlx::query_scalar!("SELECT versions FROM flow WHERE path = 'f/dre/flow'") + .fetch_one(&db) + .await + .unwrap() + ); + + // Verify cleanup - all debounce entries should be cleaned up + assert_eq!( + 0, + sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + "All debounce_key entries should be cleaned up after job completion" + ); + + assert_eq!( + 0, + sqlx::query_scalar!("SELECT COUNT(*) from debounce_stale_data") + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + "All debounce_stale_data entries should be cleaned up after job completion" + ); + + // Verify locks + { + assert_eq!( + sqlx::query_scalar!("SELECT jsonb_array_elements(value->'modules')->'value'->>'lock' AS lock FROM flow") + .fetch_all(&db) + .await + .unwrap(), + vec![ + Some("# py: 3.11\nbottle==0.13.2".into()), + Some("# py: 3.11\nbottle==0.13.2\ntiny==0.1.3".into()), + Some("# py: 3.11\ntiny==0.1.3".into()) + ] + ); + } + + Ok(()) + } + /// 2. Same as second test, however first flow djob will take longer than second debounce. + /// NOTE: This test should be ran in debug mode with `private` features enabled. In release it will not work properly. + #[cfg(all(feature = "python", feature = "private"))] + #[sqlx::test(fixtures("base", "djob_debouncing"))] + // #[windmill::all_min_versions] + async fn test_3(db: sqlx::Pool) -> anyhow::Result<()> { + // This tests checks if concurrency limit works correcly and there is no race conditions. + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = true; + } + + let (_client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + // At this point we should have two + let mut job_ids = vec![]; + let push_job = |delay, version, db, nodes_to_relock, debounce_job_id_o| async move { + let mut args = std::collections::HashMap::new(); + args.insert( + "dbg_djob_sleep".to_owned(), + // First one will create delay for 5 seconds + // The second will have no delay at all. + windmill_common::worker::to_raw_value(&delay), + ); + + args.insert( + "nodes_to_relock".to_owned(), + windmill_common::worker::to_raw_value(&nodes_to_relock), + ); + + args.insert( + "triggered_by_relative_import".to_string(), + windmill_common::worker::to_raw_value(&()), + ); + + let (job_uuid, new_tx) = windmill_queue::push( + &db, + windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), + "test-workspace", + windmill_common::jobs::JobPayload::FlowDependencies { + path: "f/dre/flow".to_owned(), + dedicated_worker: None, + // In newest versions we pass the current version to the djob + // version: 1443253234253454, + version, + }, + windmill_queue::PushArgs { args: &args, extra: None }, + "admin", + "admin@windmill.dev", + "admin".to_owned(), + Some("trigger.dependents.to.recompute.dependencies"), + // Schedule for now. + Some(chrono::Utc::now()), + None, + None, + None, + None, + None, + false, + false, + None, + true, + Some("dependency".into()), + None, + None, + None, + None, + false, + None, + debounce_job_id_o, + ) + .await + .unwrap(); + + new_tx.commit().await.unwrap(); + + job_uuid + }; + + // Push first + job_ids.push(push_job(5, 1443253234253454, db.clone(), ["a", "b"], None).await); + + // Verify debounce_stale_data and debounce_key + { + let q = sqlx::query!("SELECT COUNT(*) FROM debounce_key") + .fetch_all(&db) + .await + .unwrap(); + + // Should be single entry + assert_eq!(q.len(), 1); + } + + // Start the first one in the background + let handle = { + let mut completed = listen_for_completed_jobs(&db).await; + let db2 = db.clone(); + tokio::spawn(async move { + in_test_worker( + &db2, + // sleep(Duration::from_secs(7)), + completed.next(), // Only wait for the single job. We are going to spawn another worker for second one. + port, + ) + .await; + }) + }; + + // Wait for the job to be created and started + // This way next job is not going to be consumed by the first one. + sleep(Duration::from_secs(2)).await; + + // Push second + job_ids.push(push_job(0, 1, db.clone(), ["b", "c"], None).await); + + // Wait for the second one to finish in separate worker. + // in_test_worker(&db, completed.next(), port).await; + in_test_worker( + &db, + async { + // First job will be pulled + completed.next().await; + // However since we have concurrency limit enabled it will get rescheduled by creation of new djob. + // So we have to wait for that one as well. + completed.next().await; + }, + port, + ) + .await; + + // Wait for the first one + handle.await.unwrap(); + + // Verify locks + { + assert_eq!( + sqlx::query_scalar!("SELECT jsonb_array_elements(value->'modules')->'value'->>'lock' AS lock FROM flow") + .fetch_all(&db) + .await + .unwrap(), + vec![ + Some("# py: 3.11\nbottle==0.13.2".into()), + Some("# py: 3.11\nbottle==0.13.2\ntiny==0.1.3".into()), + Some("# py: 3.11\ntiny==0.1.3".into()) + ] + ); + } + // Verify that we have expected outcome + { + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job",) + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + 2 + ); + + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_completed",) + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + 2 + ); + // Check that two jobs were executed sequentially + assert!(sqlx::query_scalar!( + " +SELECT + j1.completed_at < j2.started_at +FROM + v2_job_completed j1, + v2_job_completed j2 +WHERE + j1.id = $1 + AND j2.id = $2", + job_ids[0], + job_ids[1], + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap()); + } + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "djob_debouncing"))] + async fn test_min_version_supports_debouncing( + db: sqlx::Pool, + ) -> anyhow::Result<()> { + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = true; + } + + let (_client, port, _s) = init_client(db.clone()).await; + let db = &db; + + crate::common::in_test_worker( + db, + async { + let job_template = crate::common::RunJob::from( + windmill_common::jobs::JobPayload::FlowDependencies { + path: "f/dre/flow".to_owned(), + dedicated_worker: None, + version: 1443253234253454, + }, + ) + .push_arg_scheduled_for_o(Some(chrono::Utc::now())) + .arg("triggered_by_relative_import", serde_json::json!(())); + + // This will push to the top level worker + let debounce_job_id = job_template.clone().push(db).await; + + // Will have space to run in parallel but in it's own worker + job_template + .push_arg_debounce_job_id_o(Some(debounce_job_id)) + .run_until_complete(db, false, port) + .await; + }, + port, + ) + .await; + + // Verify there is not jobs running + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") + .fetch_one(db) + .await + .unwrap() + .unwrap(), + 0 + ); + + // And there is only supposed to be one job. + assert_eq!( + sqlx::query_scalar!( + "SELECT COUNT(*) FROM v2_job_completed WHERE status = 'success'" + ) + .fetch_one(db) + .await + .unwrap() + .unwrap(), + 1 + ); + + Ok(()) + } + // NOTE: Don't run in parallel with other tests + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "djob_debouncing"))] + #[ignore] + async fn test_min_version_does_not_support_debouncing( + db: sqlx::Pool, + ) -> anyhow::Result<()> { + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = false; + } + + let (_client, port, _s) = init_client(db.clone()).await; + let db = &db; + + crate::common::in_test_worker( + db, + async { + let job_template = crate::common::RunJob::from( + windmill_common::jobs::JobPayload::FlowDependencies { + path: "f/dre/flow".to_owned(), + dedicated_worker: None, + version: 1443253234253454, + }, + ) + .push_arg_scheduled_for_o(Some(chrono::Utc::now())) + .arg("triggered_by_relative_import", serde_json::json!(())); + + // This will push to the top level worker + let debounce_job_id = job_template.clone().push(db).await; + + // Will have space to run in parallel but in it's own worker + job_template + .push_arg_debounce_job_id_o(Some(debounce_job_id)) + .run_until_complete(db, false, port) + .await; + }, + port, + ) + .await; + + // Verify there is not jobs running + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") + .fetch_one(db) + .await + .unwrap() + .unwrap(), + 0 + ); + + // There are supposed to be two jobs, since debouncing is disabled. + assert_eq!( + sqlx::query_scalar!( + "SELECT COUNT(*) FROM v2_job_completed WHERE status = 'success'" + ) + .fetch_one(db) + .await + .unwrap() + .unwrap(), + 2 + ); + + Ok(()) + } + // TODO: + // test that update or create flow that should bypass debouncing + } + + /// ## Testing for Apps + /// For apps we are going to do similar tests that we did for flows + mod apps { + use crate::common::{in_test_worker, init_client, listen_for_completed_jobs}; + use crate::dependency_job_debouncing::trigger_djob_for; + use std::time::Duration; + use tokio::time::sleep; + use tokio_stream::StreamExt; + + /// 1. LLF and RLF create two djobs for flow at the same and fall into single debounce + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "djob_debouncing"))] + async fn test_1(db: sqlx::Pool) -> anyhow::Result<()> { + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = true; + } + // This tests if debouncing and consolidation works. + // Also makes sures that dependency job does not create new flow version + + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job") + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + 0 + ); + + // Trigger both at the same time. + // It will create two immediate dependency jobs + { + trigger_djob_for( + &client, + "f/dre_app/leaf_left", + "0000000000069CF8", + Some("#requirements:\n#bottle==0.13.2\ndef main():\npass".into()), + ) + .await; + + trigger_djob_for( + &client, + "f/dre_app/leaf_right", + "0000000000069CFB", + Some("#requirements:\n#tiny==0.1.3\ndef main():\npass".into()), + ) + .await; + } + + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job") + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + 2 + ); + + // Spawn single worker. + in_test_worker( + &db, + async { + assert_eq!( + &sqlx::query_scalar!( + "SELECT runnable_path FROM v2_job WHERE id = $1", + completed.next().await.unwrap() + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + "f/dre_app/leaf_left" + ); + + assert_eq!( + &sqlx::query_scalar!( + "SELECT runnable_path FROM v2_job WHERE id = $1", + completed.next().await.unwrap() + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + "f/dre_app/leaf_right" + ); + + // Verify there is only one queued job that is scheduled for atleast 3s ahead. + { + let q = sqlx::query_scalar!( + "SELECT (scheduled_for - created_at) FROM v2_job_queue" + ) + .fetch_all(&db) + .await + .unwrap(); + + assert_eq!(1, q.len()); + assert!(dbg!(q[0].unwrap().microseconds) > 2_000_000); + } + + // Verify debounce_stale_data and debounce_key + { + let q = sqlx::query!( + "SELECT + dsd.to_relock, + dk.key + FROM debounce_key dk + JOIN debounce_stale_data dsd ON dk.job_id = dsd.job_id" + ) + .fetch_all(&db) + .await + .unwrap(); + + // Should be single entry + assert!(q.len() == 1); + + // This verifies that all nodes_to_relock are consolidated correctly + // AND there is no doublicats + assert_eq!( + q[0].to_relock.clone().unwrap(), + vec!["a".to_owned(), "b".to_owned(), "c".to_owned()] + ); + + // Should be workspace specific and these specific tests cover only dependency job debouncing + assert_eq!( + q[0].key.clone(), + "test-workspace:f/dre_app/app:dependency".to_owned(), + ); + } + + assert_eq!( + &sqlx::query_scalar!( + "SELECT runnable_path FROM v2_job WHERE id = $1", + completed.next().await.unwrap() + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + "f/dre_app/app" + ); + }, + port, + ) + .await; + + // Verify App states + { + let q = dbg!(sqlx::query_scalar!( + "SELECT versions FROM app WHERE path = 'f/dre_app/app'" + ) + .fetch_one(&db) + .await + .unwrap()); + + assert_eq!(2, q.len()); + + // There is also supposed to be this amount of app_versions + assert_eq!( + 2, + sqlx::query_scalar!("SELECT COUNT(*) FROM app_version WHERE app_id = '2'") + .fetch_one(&db) + .await + .unwrap() + .unwrap() + ); + } + + // Verify cleanup + { + assert_eq!( + 0, + sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") + .fetch_one(&db) + .await + .unwrap() + .unwrap() + ); + assert_eq!( + 0, + sqlx::query_scalar!("SELECT COUNT(*) from debounce_stale_data") + .fetch_one(&db) + .await + .unwrap() + .unwrap() + ); + } + + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "djob_debouncing"))] + async fn test_left(db: sqlx::Pool) -> anyhow::Result<()> { + use crate::common::RunJob; + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = true; + } + + // TODO: We don't care about timer. If there is no timer, it will be set automatically for djobs?? + let (_client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + let mut args = std::collections::HashMap::new(); + args.insert( + "dbg_djob_sleep".to_owned(), + // Execution should take this seconds + windmill_common::worker::to_raw_value(&20), + ); + args.insert( + "triggered_by_relative_import".to_owned(), + // Execution should take this seconds + windmill_common::worker::to_raw_value(&()), + ); + + let (_flow_id, new_tx) = windmill_queue::push( + &db, + windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), + "test-workspace", + windmill_common::jobs::JobPayload::AppDependencies { + path: "f/dre_app/app".to_owned(), + version: 0, + }, + windmill_queue::PushArgs { args: &args, extra: None }, + "admin", + "admin@windmill.dev", + "admin".to_owned(), + Some("trigger.dependents.to.recompute.dependencies"), + // Debounce period + Some(chrono::Utc::now() + chrono::Duration::seconds(5)), + None, + None, + None, + None, + None, + false, + false, + None, + true, + Some("dependency".into()), + None, + None, + None, + None, + false, + None, + None, + ) + .await + .unwrap(); + new_tx.commit().await.unwrap(); + + // let mut handle = { + // let mut completed = listen_for_completed_jobs(&db).await; + // let db2 = db.clone(); + // let uuid = flow_id.clone(); + // tokio::spawn(async move { + // in_test_worker( + // &db2, + // // tokio::time::sleep(tokio::time::Duration::from_secs(60)), + // async move { + // completed.find(&uuid).await; + // }, + // port, + // ) + // .await; + // }) + // }; + + let db2 = db.clone(); + in_test_worker( + &db2, + async { + // This job should execute and then try to start another job that will get debounced. + RunJob::from(windmill_common::jobs::JobPayload::Dependencies { + path: "f/dre_app/leaf_right".to_owned(), + hash: 433403.into(), + language: windmill_common::scripts::ScriptLang::Python3, + dedicated_worker: None, + }) + // .arg("dbg_djob_sleep", serde_json::json!(10)) + .run_until_complete(&db, false, port) + .await; + + RunJob::from(windmill_common::jobs::JobPayload::Dependencies { + path: "f/dre_app/leaf_left".to_owned(), + hash: 433400.into(), + language: windmill_common::scripts::ScriptLang::Python3, + dedicated_worker: None, + }) + // So set it to this long + .arg("dbg_djob_sleep", serde_json::json!(10)) + .run_until_complete(&db, false, port) + .await; + + completed.next().await; // leaf_right + completed.next().await; // leaf_left + completed.next().await; // importer + completed.next().await; // importer + }, + port, + ) + .await; + + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + 0 + ); + + let r = sqlx::query_scalar!("SELECT runnable_id FROM v2_job ORDER BY created_at DESC") + .fetch_all(&db) + .await + .unwrap(); + + assert_eq!(r.len(), 4); + assert!(r.contains(&Some(9))); + assert!(r.contains(&Some(433400))); + assert!(r.contains(&Some(433403))); + assert!(r.contains(&Some(0))); + + // handle.await.unwrap(); + + Ok(()) + } + /// 2. Same as second test, however first app djob will take longer than second debounce. + /// NOTE: This test should be ran in debug mode. In release it will not work properly. + #[cfg(all(feature = "python", feature = "private"))] + #[sqlx::test(fixtures("base", "djob_debouncing"))] + async fn test_3(db: sqlx::Pool) -> anyhow::Result<()> { + // This tests checks if concurrency limit works correcly and there is no race conditions. + let (_client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + // At this point we should have two + let mut job_ids = vec![]; + let push_job = |delay, db| async move { + let mut args = std::collections::HashMap::new(); + args.insert( + "dbg_djob_sleep".to_owned(), + // First one will create delay for 5 seconds + // The second will have no delay at all. + windmill_common::worker::to_raw_value(&delay), + ); + + args.insert( + "triggered_by_relative_import".to_string(), + windmill_common::worker::to_raw_value(&()), + ); + + let (job_uuid, new_tx) = windmill_queue::push( + &db, + windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), + "test-workspace", + windmill_common::jobs::JobPayload::AppDependencies { + path: "f/dre_app/app".to_owned(), + // In newest versions we pass the current version to the djob + version: 0, + }, + windmill_queue::PushArgs { args: &args, extra: None }, + "admin", + "admin@windmill.dev", + "admin".to_owned(), + Some("trigger.dependents.to.recompute.dependencies"), + // Schedule for now. + Some(chrono::Utc::now()), + None, + None, + None, + None, + None, + false, + false, + None, + true, + Some("dependency".into()), + None, + None, + None, + None, + false, + None, + None, + ) + .await + .unwrap(); + + new_tx.commit().await.unwrap(); + + job_uuid + }; + + // TODO: Verify concurrency key. + // Push first + job_ids.push(push_job(5, db.clone()).await); + + // Verify debounce_stale_data and debounce_key + { + let q = sqlx::query!("SELECT COUNT(*) FROM debounce_key") + .fetch_all(&db) + .await + .unwrap(); + + // Should be single entry + assert_eq!(q.len(), 1); + } + + // Start the first one in the background + let handle = { + let mut completed = listen_for_completed_jobs(&db).await; + let db2 = db.clone(); + tokio::spawn(async move { + in_test_worker( + &db2, + // sleep(Duration::from_secs(7)), + completed.next(), // Only wait for the single job. We are going to spawn another worker for second one. + port, + ) + .await; + }) + }; + + // Wait for the job to be created and started + // This way next job is not going to be consumed by the first one. + sleep(Duration::from_secs(2)).await; + + // Push second + job_ids.push(push_job(0, db.clone()).await); + + // Wait for the second one to finish in separate worker. + // in_test_worker(&db, completed.next(), port).await; + in_test_worker( + &db, + async { + // First job will be pulled + completed.next().await; + // However since we have concurrency limit enabled it will get rescheduled by creation of new djob. + // So we have to wait for that one as well. + completed.next().await; + }, + port, + ) + .await; + + // Wait for the first one + handle.await.unwrap(); + + // Verify that we have expected outcome + { + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job",) + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + 2 + ); + + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_completed",) + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + 2 + ); + // Check that two jobs were executed sequentially + assert!(sqlx::query_scalar!( + " +SELECT + j1.completed_at < j2.started_at +FROM + v2_job_completed j1, + v2_job_completed j2 +WHERE + j1.id = $1 + AND j2.id = $2", + job_ids[0], + job_ids[1], + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap()); + } + Ok(()) + } + } + + // TODO: Test debounce reassignment works + + /// ## Testing for Scripts + mod scripts { + use crate::common::{in_test_worker, init_client, listen_for_completed_jobs}; + use crate::dependency_job_debouncing::trigger_djob_for; + use std::time::Duration; + use tokio::time::sleep; + use tokio_stream::StreamExt; + + /// 1. LLF and RLF create two djobs for flow at the same and fall into single debounce + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "djob_debouncing"))] + // TODO: Same test_but script fails. + async fn test_1(db: sqlx::Pool) -> anyhow::Result<()> { + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = true; + } + // This tests if debouncing and consolidation works. + // Also makes sures that dependency job does not create new flow version + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + // Verify lock is empty + { + assert_eq!( + sqlx::query_scalar!( + "SELECT lock FROM script WHERE path = 'f/dre_script/script'" + ) + .fetch_one(&db) + .await + .unwrap(), + Some("".into()) + ); + } + + // Trigger both at the same time. + { + trigger_djob_for( + &client, + "f/dre_script/leaf_left", + "0000000000082398", + Some("#requirements:\n#bottle==0.13.2\ndef main():\npass".into()), + ) + .await; + trigger_djob_for( + &client, + "f/dre_script/leaf_right", + "000000000008239B", + Some("#requirements:\n#tiny==0.1.3\ndef main():\npass".into()), + ) + .await; + } + + sleep(Duration::from_secs(1)).await; + + in_test_worker( + &db, + async { + assert_eq!( + &sqlx::query_scalar!( + "SELECT runnable_path FROM v2_job WHERE id = $1", + completed.next().await.unwrap() + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + "f/dre_script/leaf_left" + ); + + assert_eq!( + &sqlx::query_scalar!( + "SELECT runnable_path FROM v2_job WHERE id = $1", + completed.next().await.unwrap() + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + "f/dre_script/leaf_right" + ); + + // handle.await.unwrap(); + + // Let jobs propagate + + tokio::select!( + _ = async { + while sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue WHERE running = false") + .fetch_one(&db) + .await + .unwrap() + .unwrap() + == 0 + { + sleep(Duration::from_secs(1)).await; + } + } => {}, + _ = sleep(Duration::from_secs(60)) => { panic!("Timeout") } + ); + // Verify there is only one queued job that is scheduled for atleast 3s ahead. + { + for r in + sqlx::query_scalar!("SELECT id FROM v2_job_queue WHERE running = false") + .fetch_all(&db) + .await + .unwrap() + { + dbg!( + sqlx::query!("SELECT runnable_path FROM v2_job WHERE id = $1", r) + .fetch_all(&db) + .await + .unwrap() + ); + } + for r in sqlx::query_scalar!("SELECT id FROM v2_job_completed") + .fetch_all(&db) + .await + .unwrap() + { + dbg!( + sqlx::query!("SELECT runnable_path FROM v2_job WHERE id = $1", r) + .fetch_all(&db) + .await + .unwrap() + ); + } + + dbg!(sqlx::query!("SELECT runnable_path FROM v2_job") + .fetch_all(&db) + .await + .unwrap()); + + let q = sqlx::query_scalar!( + "SELECT (scheduled_for - created_at) FROM v2_job_queue WHERE running = false" + ) + .fetch_all(&db) + .await + .unwrap(); + + assert_eq!(1, q.len()); + assert!(dbg!(q[0].unwrap().microseconds) > 1_000_000 /* 1 second */); + } + + // Verify debounce_stale_data and debounce_key + { + let q = sqlx::query_scalar!("SELECT key FROM debounce_key") + .fetch_all(&db) + .await + .unwrap(); + + assert_eq!(q.len(), 1); + + assert_eq!( + q[0].clone(), + "test-workspace:f/dre_script/script:dependency".to_owned(), + ); + + // Stale data is empty for scripts + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM debounce_stale_data") + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + 0 + ); + } + + // Wait until debounce delay is complete + // sleep(Duration::from_secs(6)).await; + + assert_eq!( + &sqlx::query_scalar!( + "SELECT runnable_path FROM v2_job WHERE id = $1", + completed.next().await.unwrap() + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + "f/dre_script/script" + ); + }, + port, + ) + .await; + + // completed.next().await.unwrap(); + + // Verify + { + assert_eq!( + 0, + sqlx::query_scalar!("SELECT COUNT(*) from v2_job_queue") + .fetch_one(&db) + .await + .unwrap() + .unwrap() + ); + assert_eq!( + vec![533404], + dbg!(sqlx::query_scalar!( + "SELECT hash FROM script WHERE path = 'f/dre_script/script' AND archived = true" + ) + .fetch_all(&db) + .await + .unwrap()) + ); + + assert_ne!( + 533404, + sqlx::query_scalar!( + "SELECT hash FROM script WHERE path = 'f/dre_script/script' AND archived = false" + ) + .fetch_one(&db) + .await + .unwrap() + ); + + assert_eq!( + vec![533404], + sqlx::query_scalar!( + "SELECT parent_hashes FROM script WHERE path = 'f/dre_script/script' AND archived = false" + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap() + ); + } + + // Verify cleanup + { + assert_eq!( + 0, + sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") + .fetch_one(&db) + .await + .unwrap() + .unwrap() + ); + assert_eq!( + 0, + sqlx::query_scalar!("SELECT COUNT(*) from debounce_stale_data") + .fetch_one(&db) + .await + .unwrap() + .unwrap() + ); + } + + // handle.await.unwrap(); + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "djob_debouncing"))] + async fn test_left(db: sqlx::Pool) -> anyhow::Result<()> { + use crate::common::RunJob; + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = true; + } + + // TODO: We don't care about timer. If there is no timer, it will be set automatically for djobs?? + let (_client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + // Trigger both at the same time. + { + let mut args = std::collections::HashMap::new(); + args.insert( + "dbg_djob_sleep".to_owned(), + // Execution should take this seconds + windmill_common::worker::to_raw_value(&20), + ); + + args.insert( + "triggered_by_relative_import".to_owned(), + // Execution should take this seconds + windmill_common::worker::to_raw_value(&()), + ); + + let (_flow_id, new_tx) = windmill_queue::push( + &db, + windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), + "test-workspace", + windmill_common::jobs::JobPayload::Dependencies { + path: "f/dre_script/script".to_owned(), + dedicated_worker: None, + language: windmill_common::scripts::ScriptLang::Python3, + hash: 533404.into(), + }, + windmill_queue::PushArgs { args: &args, extra: None }, + "admin", + "admin@windmill.dev", + "admin".to_owned(), + Some("trigger.dependents.to.recompute.dependencies"), + // Debounce period + Some(chrono::Utc::now() + chrono::Duration::seconds(5)), + None, + None, + None, + None, + None, + false, + false, + None, + true, + Some("dependency".into()), + None, + None, + None, + None, + false, + None, + None, + ) + .await + .unwrap(); + new_tx.commit().await.unwrap(); + + let db2 = db.clone(); + in_test_worker( + &db2, + async { + // This job should execute and then try to start another job that will get debounced. + RunJob::from(windmill_common::jobs::JobPayload::Dependencies { + path: "f/dre_script/leaf_right".to_owned(), + hash: 533403.into(), + language: windmill_common::scripts::ScriptLang::Python3, + dedicated_worker: None, + }) + .run_until_complete(&db, false, port) + .await; + + // This one is supposed to be started after flow djob has debounced and started but haven't finished yet. + RunJob::from(windmill_common::jobs::JobPayload::Dependencies { + path: "f/dre_script/leaf_left".to_owned(), + hash: 533400.into(), + language: windmill_common::scripts::ScriptLang::Python3, + dedicated_worker: None, + }) + // So set it to this long + .arg("dbg_djob_sleep", serde_json::json!(10)) + .run_until_complete(&db, false, port) + .await; + + completed.next().await; // leaf_right + completed.next().await; // leaf_left + completed.next().await; // importer + completed.next().await; // importer + }, + port, + ) + .await; + } + + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") + .fetch_one(&db) + .await + .unwrap() + .unwrap(), + 0 + ); + + let r = sqlx::query_scalar!("SELECT runnable_id FROM v2_job ORDER BY created_at DESC") + .fetch_all(&db) + .await + .unwrap(); + + dbg!(&r); + assert_eq!(r.len(), 4); + assert!(r.contains(&Some(-221349019907577876))); + assert!(r.contains(&Some(533400))); + assert!(r.contains(&Some(533403))); + assert!(r.contains(&Some(533404))); + + Ok(()) + } + + // // TODO: we don't need scripts to have concurrency limit + // /// 3. Same as second test, however first app djob will take longer than second debounce. + // /// NOTE: This test should be ran in debug mode. In release it will not work properly. + // #[cfg(all(feature = "python", feature = "private"))] + // #[sqlx::test(fixtures("base", "djob_debouncing"))] + // async fn test_3(db: sqlx::Pool) -> anyhow::Result<()> { + // // This tests checks if concurrency limit works correcly and there is no race conditions. + // let (client, port, _s) = init_client(db.clone()).await; + // let mut completed = listen_for_completed_jobs(&db).await; + + // // At this point we should have two + // let mut job_ids = vec![]; + // let push_job = |delay, db| async move { + // let mut args = std::collections::HashMap::new(); + // args.insert( + // "dbg_djob_sleep".to_owned(), + // // First one will create delay for 5 seconds + // // The second will have no delay at all. + // windmill_common::worker::to_raw_value(&delay), + // ); + + // args.insert( + // "triggered_by_relative_import".to_string(), + // windmill_common::worker::to_raw_value(&()), + // ); + + // let (job_uuid, new_tx) = windmill_queue::push( + // &db, + // windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), + // "test-workspace", + // windmill_common::jobs::JobPayload::Dependencies { + // path: "f/dre_script/script".to_owned(), + // language: windmill_common::scripts::ScriptLang::Python3, + // dedicated_worker: None, + // hash: windmill_common::scripts::ScriptHash(533404), + // }, + // windmill_queue::PushArgs { args: &args, extra: None }, + // "admin", + // "admin@windmill.dev", + // "admin".to_owned(), + // Some("trigger.dependents.to.recompute.dependencies"), + // // Schedule for now. + // Some(chrono::Utc::now()), + // None, + // None, + // None, + // None, + // None, + // false, + // false, + // None, + // true, + // Some("dependency".into()), + // None, + // None, + // None, + // None, + // false, + // None, + // None, + // ) + // .await + // .unwrap(); + + // new_tx.commit().await.unwrap(); + + // job_uuid + // }; + + // // Push first + // job_ids.push(push_job(5, db.clone()).await); + // sleep(Duration::from_millis(300)).await; + + // // Verify debounce_stale_data and debounce_key + // { + // let q = sqlx::query_scalar!("SELECT key FROM debounce_key") + // .fetch_all(&db) + // .await + // .unwrap(); + + // assert_eq!(q.len(), 1); + + // assert_eq!( + // q[0].clone(), + // "test-workspace:f/dre_script/script:dependency".to_owned(), + // ); + + // // Stale data is empty for scripts + // assert_eq!( + // sqlx::query_scalar!("SELECT COUNT(*) FROM debounce_stale_data") + // .fetch_one(&db) + // .await + // .unwrap() + // .unwrap(), + // 0 + // ); + // } + + // // Start the first one in the background + // let handle = { + // let mut completed = listen_for_completed_jobs(&db).await; + // let db2 = db.clone(); + // tokio::spawn(async move { + // in_test_worker( + // &db2, + // // sleep(Duration::from_secs(7)), + // completed.next(), // Only wait for the single job. We are going to spawn another worker for second one. + // port, + // ) + // .await; + // }) + // }; + + // // Wait for the job to be created and started + // // This way next job is not going to be consumed by the first one. + // sleep(Duration::from_secs(1)).await; + + // // Push second + // job_ids.push(push_job(0, db.clone()).await); + + // // Wait for the second one to finish in separate worker. + // in_test_worker( + // &db, + // async { + // // First job will be pulled + // completed.next().await; + // // However since we have concurrency limit enabled it will get rescheduled by creation of new djob. + // // So we have to wait for that one as well. + // completed.next().await; + // }, + // port, + // ) + // .await; + + // // Wait for the first one + // handle.await.unwrap(); + + // // Verify that we have expected outcome + // { + // assert_eq!( + // sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue",) + // .fetch_one(&db) + // .await + // .unwrap() + // .unwrap(), + // 0 + // ); + // // Verify lock + // { + // assert_eq!( + // sqlx::query_scalar!( + // "SELECT lock FROM script WHERE path = 'f/dre_script/script'" + // ) + // .fetch_one(&db) + // .await + // .unwrap(), + // Some("# py: 3.11\nbottle==0.13.2\ntiny==0.1.3".into()) + // ); + // } + // assert_eq!( + // sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job",) + // .fetch_one(&db) + // .await + // .unwrap() + // .unwrap(), + // 2 + // ); + + // assert_eq!( + // sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_completed",) + // .fetch_one(&db) + // .await + // .unwrap() + // .unwrap(), + // 2 + // ); + // // Check that two jobs were executed sequentially + // assert!(sqlx::query_scalar!( + // " + // SELECT + // j1.completed_at < j2.started_at + // FROM + // v2_job_completed j1, + // v2_job_completed j2 + // WHERE + // j1.id = $1 + // AND j2.id = $2", + // job_ids[0], + // job_ids[1], + // ) + // .fetch_one(&db) + // .await + // .unwrap() + // .unwrap()); + // } + // Ok(()) + // } + } + // TODO: Test git sync +} +#[cfg(feature = "test_job_debouncing")] +mod normal_job_debouncing { + mod scripts { + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "job_debouncing"))] + async fn test_default_debounce_key(db: sqlx::Pool) -> anyhow::Result<()> { + use serde_json::json; + use windmill_common::scripts::ScriptHash; + + use crate::common::{in_test_worker, init_client, listen_for_completed_jobs, RunJob}; + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = true; + } + + let (_client, port, _s) = init_client(db.clone()).await; + let db = &db; + + in_test_worker( + db, + async { + // This job should execute and then try to start another job that will get debounced. + RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { + hash: ScriptHash(533400), + path: "f/scripts/script_1".into(), + // Do not supply with custom debounce key. + // We will test if the debounce_key is created correctly. + custom_debounce_key: None, + debounce_delay_s: Some(2), + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + language: windmill_common::scripts::ScriptLang::Python3, + priority: None, + apply_preprocessor: false, + }) + .arg("x", json!("ey")) + .arg("b", json!("33")) + // Start another worker, so we have two workers at the same time. + // We don't know which will execute the job, but we do know that if the job is executed, this worker will exit. + .run_until_complete_with(db, false, port, |id| async move { + + // Verify debounce_key + assert_eq!( + sqlx::query_scalar!( + "SELECT key FROM debounce_key WHERE job_id = $1", + id.clone() + ) + .fetch_one(db) + .await + .unwrap(), + "test-workspace/script/f/scripts/script_1#args:\"33\":\"ey\"".to_owned() + ); + + // Verify it is scheduled for future and not now. + { + assert!( + dbg!( + sqlx::query_scalar!( + "SELECT (scheduled_for - created_at) FROM v2_job_queue WHERE running = false" + ) + .fetch_one(db) + .await + .unwrap() + .unwrap() + .microseconds + ) > 1_000_000 /* 1 second */ + ); + } + + // Start another job. + RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { + hash: ScriptHash(533400), + path: "f/scripts/script_1".into(), + // Do not supply with custom debounce key. + // We will test if the debounce_key is created correctly. + custom_debounce_key: None, + debounce_delay_s: Some(2), + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + language: windmill_common::scripts::ScriptLang::Python3, + priority: None, + apply_preprocessor: false, + }) + .arg("x", json!("ey")) + .arg("b", json!("33")) + // But we only push it, one of the jobs should be debounced. + .push(db) + .await; + }) + .await; + }, + port, + ) + .await; + + // Verify there is not jobs running + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") + .fetch_one(db) + .await + .unwrap() + .unwrap(), + 0 + ); + + // And there is only supposed to be one job. + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job") + .fetch_one(db) + .await + .unwrap() + .unwrap(), + 1 + ); + + // Verify debounce key clean up + assert_eq!( + 0, + sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") + .fetch_one(db) + .await + .unwrap() + .unwrap() + ); + + Ok(()) + } + + #[cfg(all(feature = "python", feature = "agent_worker_server"))] + #[sqlx::test(fixtures("base", "job_debouncing"))] + async fn test_default_debounce_key_agent_wk( + db: sqlx::Pool, + ) -> anyhow::Result<()> { + use serde_json::json; + use windmill_common::scripts::ScriptHash; + + use crate::common::{init_client_agent_mode, RunJob}; + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = true; + } + + let (_client, port, _s) = init_client_agent_mode(db.clone()).await; + let db = &db; + RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { + hash: ScriptHash(533400), + path: "f/scripts/script_1".into(), + // Do not supply with custom debounce key. + // We will test if the debounce_key is created correctly. + custom_debounce_key: None, + debounce_delay_s: Some(2), + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + language: windmill_common::scripts::ScriptLang::Python3, + priority: None, + apply_preprocessor: false, + }) + .arg("x", json!("ey")) + .arg("b", json!("33")) + .run_until_complete(db, true, port) + .await; + + // Verify there is not jobs running + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") + .fetch_one(db) + .await + .unwrap() + .unwrap(), + 0 + ); + + // And there is only supposed to be one job. + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job") + .fetch_one(db) + .await + .unwrap() + .unwrap(), + 1 + ); + + // And that job execute successfully + assert_eq!( + sqlx::query_scalar!("SELECT status::text FROM v2_job_completed") + .fetch_one(db) + .await + .unwrap() + .unwrap(), + "success" + ); + + // Verify debounce key clean up + assert_eq!( + 0, + sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") + .fetch_one(db) + .await + .unwrap() + .unwrap() + ); + + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "job_debouncing"))] + async fn test_custom_debounce_key(db: sqlx::Pool) -> anyhow::Result<()> { + use serde_json::json; + use windmill_common::scripts::ScriptHash; + + use crate::common::{in_test_worker, init_client, listen_for_completed_jobs, RunJob}; + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = true; + } + + let (_client, port, _s) = init_client(db.clone()).await; + let completed = listen_for_completed_jobs(&db).await; + let db = &db; + + in_test_worker( + db, + async { + // This job should execute and then try to start another job that will get debounced. + RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { + hash: ScriptHash(533400), + path: "f/scripts/script_1".into(), + // Do not supply with custom debounce key. + // We will test if the debounce_key is created correctly. + custom_debounce_key: Some("$workspace:my-custom-debounce-key:$args[x]".to_owned()), + debounce_delay_s: Some(2), + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + language: windmill_common::scripts::ScriptLang::Python3, + priority: None, + apply_preprocessor: false, + }) + .arg("x", json!("ey")) + .arg("b", json!("1")) // 1 + // Start another worker, so we have two workers at the same time. + // We don't know which will execute the job, but we do know that if the job is executed, this worker will exit. + .run_until_complete_with(db, false, port, |id| async move { + + // Verify debounce_key + assert_eq!( + sqlx::query_scalar!( + "SELECT key FROM debounce_key WHERE job_id = $1", + id.clone() + ) + .fetch_one(db) + .await + .unwrap(), + "test-workspace:my-custom-debounce-key:ey".to_owned() + ); + + // Verify it is scheduled for future and not now. + { + assert!( + dbg!( + sqlx::query_scalar!( + "SELECT (scheduled_for - created_at) FROM v2_job_queue WHERE running = false" + ) + .fetch_one(db) + .await + .unwrap() + .unwrap() + .microseconds + ) > 1_000_000 /* 1 second */ + ); + } + + // Start another job. + RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { + hash: ScriptHash(533400), + path: "f/scripts/script_1".into(), + custom_debounce_key: Some("$workspace:my-custom-debounce-key:$args[x]".to_owned()), + debounce_delay_s: Some(2), + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + language: windmill_common::scripts::ScriptLang::Python3, + priority: None, + apply_preprocessor: false, + }) + .arg("x", json!("ey")) + // We will pass different argument. but it should still get debounced. + .arg("b", json!("2")) // 2 + // But we only push it, one of the jobs should be debounced. + .push(db) + .await; + }) + .await; + }, + port, + ) + .await; + + // Verify there is not jobs running + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") + .fetch_one(db) + .await + .unwrap() + .unwrap(), + 0 + ); + + // And there is only supposed to be one job. + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job") + .fetch_one(db) + .await + .unwrap() + .unwrap(), + 1 + ); + + // And that job execute successfully + assert_eq!( + sqlx::query_scalar!("SELECT status::text FROM v2_job_completed") + .fetch_one(db) + .await + .unwrap() + .unwrap(), + "success" + ); + + // Verify debounce key clean up + assert_eq!( + 0, + sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") + .fetch_one(db) + .await + .unwrap() + .unwrap() + ); + + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "job_debouncing"))] + async fn test_no_debounce(db: sqlx::Pool) -> anyhow::Result<()> { + use serde_json::json; + use windmill_common::scripts::ScriptHash; + + use crate::common::{in_test_worker, init_client, listen_for_completed_jobs, RunJob}; + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = true; + } + + let (_client, port, _s) = init_client(db.clone()).await; + let completed = listen_for_completed_jobs(&db).await; + let db = &db; + + // different args + in_test_worker( + db, + async { + // This job should execute and then try to start another job that will get debounced. + RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { + hash: ScriptHash(533400), + path: "f/scripts/script_1".into(), + // Do not supply with custom debounce key. + // We will test if the debounce_key is created correctly. + custom_debounce_key: None, + debounce_delay_s: Some(2), + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + language: windmill_common::scripts::ScriptLang::Python3, + priority: None, + apply_preprocessor: false, + }) + .arg("x", json!("ey")) + .arg("b", json!("33")) + // Start another worker, so we have two workers at the same time. + // We don't know which will execute the job, but we do know that if the job is executed, this worker will exit. + .run_until_complete_with(db, false, port, |_id| async move { + // Start another job. + RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { + hash: ScriptHash(533400), + path: "f/scripts/script_1".into(), + // Do not supply with custom debounce key. + // We will test if the debounce_key is created correctly. + custom_debounce_key: None, + debounce_delay_s: Some(2), + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + language: windmill_common::scripts::ScriptLang::Python3, + priority: None, + apply_preprocessor: false, + }) + // Different args. + .arg("x", json!("ey")) + .arg("b", json!("34")) // Different arg + .push(db) + .await; + }) + .await; + }, + port, + ) + .await; + + // no debounce delay on second + in_test_worker( + db, + async { + // This job should execute and then try to start another job that will get debounced. + RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { + hash: ScriptHash(533400), + path: "f/scripts/script_1".into(), + // Do not supply with custom debounce key. + // We will test if the debounce_key is created correctly. + custom_debounce_key: None, + debounce_delay_s: Some(2), + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + language: windmill_common::scripts::ScriptLang::Python3, + priority: None, + apply_preprocessor: false, + }) + .arg("x", json!("ey")) + .arg("b", json!("33")) + // Start another worker, so we have two workers at the same time. + // We don't know which will execute the job, but we do know that if the job is executed, this worker will exit. + .run_until_complete_with(db, false, port, |_id| async move { + // Start another job. + RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { + hash: ScriptHash(533400), + path: "f/scripts/script_1".into(), + custom_debounce_key: None, + debounce_delay_s: None, // Set to none to skip debouncing + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + language: windmill_common::scripts::ScriptLang::Python3, + priority: None, + apply_preprocessor: false, + }) + .arg("x", json!("ey")) + .arg("b", json!("33")) + .push(db) + .await; + }) + .await; + }, + port, + ) + .await; + + // Verify there is not jobs running + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") + .fetch_one(db) + .await + .unwrap() + .unwrap(), + 0 + ); + + // And there is supposed to be four jobs and no debouncing. + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job") + .fetch_one(db) + .await + .unwrap() + .unwrap(), + 4 + ); + + // Verify debounce key clean up + assert_eq!( + 0, + sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") + .fetch_one(db) + .await + .unwrap() + .unwrap() + ); + + Ok(()) + } + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "djob_debouncing"))] + #[ignore = "modifies global env variable that is used by other tests"] + async fn test_min_version_does_not_support_debouncing( + db: sqlx::Pool, + ) -> anyhow::Result<()> { + use serde_json::json; + use windmill_common::scripts::ScriptHash; + + use crate::common::{in_test_worker, init_client, listen_for_completed_jobs, RunJob}; + + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = false; + } + + let (_client, port, _s) = init_client(db.clone()).await; + let db = &db; + + crate::common::in_test_worker( + db, + async { + let job_template = + RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { + hash: ScriptHash(533400), + path: "f/scripts/script_1".into(), + custom_debounce_key: None, + debounce_delay_s: None, // Set to none to skip debouncing + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + language: windmill_common::scripts::ScriptLang::Python3, + priority: None, + apply_preprocessor: false, + }); + + // This will push to the top level worker + job_template.clone().push(db).await; + + // Will have space to run in parallel but in it's own worker + job_template.run_until_complete(db, false, port).await; + }, + port, + ) + .await; + + // Verify there is not jobs running + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") + .fetch_one(db) + .await + .unwrap() + .unwrap(), + 0 + ); + + // There are supposed to be two jobs, since debouncing is disabled. + assert_eq!( + sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job") + .fetch_one(db) + .await + .unwrap() + .unwrap(), + 2 + ); + + Ok(()) + } + } + + mod flows { + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "job_debouncing"))] + async fn test_different_kinds_top_level( + db: sqlx::Pool, + ) -> anyhow::Result<()> { + use crate::common::{init_client, listen_for_completed_jobs, RunJob}; + use serde_json::json; + { + let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING + .write() + .await; + *mvsd = true; + } + + let (_client, port, _s) = init_client(db.clone()).await; + let db = &db; + + // We want to run this for all tables related to flow be created. + RunJob::from(windmill_common::jobs::JobPayload::FlowDependencies { + path: "f/flows/flow".into(), + dedicated_worker: None, + version: 1443253234253454, + }) + .run_until_complete(db, false, port) + .await; + + dbg!(sqlx::query!("SELECT * FROM flow_node",) + .fetch_all(db) + .await + .unwrap()); + + let (j1, j2, j3) = tokio::join!( + RunJob::from(windmill_common::jobs::JobPayload::Flow { + version: 1443253234253454, + path: "f/flows/flow".into(), + dedicated_worker: None, + apply_preprocessor: false, + }) + .push(db), + RunJob::from(windmill_common::jobs::JobPayload::SingleStepFlow { + hash: None, + path: "f/flows/flow".into(), + custom_debounce_key: None, + debounce_delay_s: Some(2), + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + flow_version: Some(1443253234253454), + args: std::collections::HashMap::new(), + retry: None, + error_handler_path: None, + error_handler_args: None, + skip_handler: None, + cache_ttl: None, + priority: None, + tag_override: None, + trigger_path: None, + apply_preprocessor: false, + }) + .push(db), + RunJob::from(windmill_common::jobs::JobPayload::RawFlow { + value: windmill_common::flows::FlowValue { + debounce_delay_s: Some(2), + modules: vec![windmill_common::flows::FlowModule { + id: "a".into(), + value: windmill_common::worker::to_raw_value(&json!({ + "lock": "# py: 3.11\n", + "type": "rawscript", + "assets": [], + "content": "def main(x: str, y: str):\n return x", + "language": "python3", + "debounce_delay_s": 15, + "input_transforms": { + "x": { + "type": "static", + "value": "" + }, + "y": { + "type": "static", + "value": "" + } + } + })), + ..Default::default() + }], + ..Default::default() + }, + path: Some("f/flows/flow".into()), + restarted_from: None, + }) + .push(db), + // RunJob::from(windmill_common::jobs::JobPayload::Code( + // windmill_common::jobs::RawCode { + // content: " + // def main(n: int): + // pass + // " + // .into(), + // path: Some("f/flows/flow".into()), + // hash: None, + // language: windmill_common::scripts::ScriptLang::Python3, + // custom_debounce_key: None, + // debounce_delay_s: Some(2), + // ..Default::default() + // }, + // )) + // .push(db) + ); + + assert_eq!(j1, j2); + assert_eq!(j1, j3); + // assert_eq!(j1, j4); + Ok(()) + } + } + + // TODO(ALL): + // - Check if all jobs were sucessfull. + // + // TODO: + // - [x] FlowNode (Script) + // - [x] FlowNode (Flow) - has no debouncing nor concurrency limits + // - [x] RawCode (Flow as code) + // - [x] RawFlow + // - [x] Flow + // - [x] FlowScript + // + // TODO(imperatively): + // - [x] Creation of flow + // - [x] Check entire flow + // - [x] Check it's inline scripts + // - [x] Creation of script + // + // TODO: [x] Agent workers. (and tests) + // TODO: [x] Backwards compat (and tests) + // TODO: [x] Concurrency limit is disabled if preprocessor is enabled. Investigate. + // TODO: [x] Last resort - monitor.rs to clean up debounce_keys + // TODO: [x] Catch debounce values by server if debouncing is disabled. (and tests) +} diff --git a/backend/tests/retry.rs b/backend/tests/retry.rs index f7162e9a45..60b81573db 100644 --- a/backend/tests/retry.rs +++ b/backend/tests/retry.rs @@ -2,13 +2,13 @@ mod common; #[cfg(feature = "deno_core")] mod retry { + use crate::common::*; use serde_json::json; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; use sqlx::{Pool, Postgres}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use windmill_common::flow_status::FlowStatusModule; use windmill_common::flows::FlowValue; use windmill_common::jobs::JobPayload; - use windmill_common::flow_status::FlowStatusModule; - use crate::common::*; pub async fn initialize_tracing() { use std::sync::Once; @@ -166,7 +166,7 @@ def main(last, port): }) .arg("items", json!(["unused", "unused", "unused"])) .arg("port", json!(server.addr.port())) - .run_until_complete(&db, server.addr.port()) + .run_until_complete(&db, false, server.addr.port()) .await .json_result() .unwrap(); @@ -201,7 +201,7 @@ def main(last, port): }) .arg("items", json!(["unused", "unused", "unused"])) .arg("port", json!(server.addr.port())) - .run_until_complete(&db, server.addr.port()) + .run_until_complete(&db, false, server.addr.port()) .await .json_result() .unwrap(); @@ -248,7 +248,7 @@ def main(last, port): }) .arg("items", json!(["unused", "unused", "unused"])) .arg("port", json!(server.addr.port())) - .run_until_complete(&db, server.addr.port()) + .run_until_complete(&db, false, server.addr.port()) .await; let result = job.json_result().unwrap(); @@ -315,7 +315,7 @@ def main(error, port): let server = Server::start(responses).await; let cjob = RunJob::from(JobPayload::RawFlow { value, path: None, restarted_from: None }) .arg("port", json!(server.addr.port())) - .run_until_complete(&db, server.addr.port()) + .run_until_complete(&db, false, server.addr.port()) .await; let result = cjob.json_result().clone().unwrap(); let failed_module = get_module(&cjob, "a").unwrap(); diff --git a/backend/tests/suspend_resume.rs b/backend/tests/suspend_resume.rs index e51aff9060..9c50488060 100644 --- a/backend/tests/suspend_resume.rs +++ b/backend/tests/suspend_resume.rs @@ -6,13 +6,13 @@ mod suspend_resume { #[cfg(feature = "deno_core")] use crate::common::*; - + #[cfg(feature = "deno_core")] - use sqlx::{Pool, Postgres}; + use futures::{Stream, StreamExt}; #[cfg(feature = "deno_core")] use sqlx::types::Uuid; #[cfg(feature = "deno_core")] - use futures::{Stream, StreamExt}; + use sqlx::{Pool, Postgres}; #[cfg(feature = "deno_core")] use windmill_common::flows::FlowValue; #[cfg(feature = "deno_core")] @@ -223,7 +223,7 @@ mod suspend_resume { .arg("n", json!(1)) .arg("op", json!("cancel")) .arg("port", json!(port)) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 379c92606b..72b6aeea52 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -1,6 +1,5 @@ use serde::de::DeserializeOwned; - #[cfg(feature = "enterprise")] use chrono::Timelike; @@ -20,20 +19,19 @@ use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs}; use windmill_common::flows::InputTransform; #[cfg(any(feature = "python", feature = "deno_core"))] -use windmill_common::flow_status::{RestartedFrom}; - +use windmill_common::flow_status::RestartedFrom; use windmill_common::{ - flows::{ FlowValue}, - jobs::{ JobPayload, RawCode}, - scripts::{ScriptLang}, - + flows::FlowValue, + jobs::{JobPayload, RawCode}, + scripts::ScriptLang, }; mod common; use common::*; #[cfg(feature = "enterprise")] use futures::StreamExt; - +use windmill_common::flows::FlowModule; +use windmill_common::flows::FlowModuleValue; // async fn _print_job(id: Uuid, db: &Pool) -> Result<(), anyhow::Error> { // tracing::info!( @@ -45,7 +43,6 @@ use futures::StreamExt; // Ok(()) // } - #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_iteration(db: Pool) -> anyhow::Result<()> { @@ -80,7 +77,7 @@ async fn test_iteration(db: Pool) -> anyhow::Result<()> { let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }) .arg("items", json!([])) - .run_until_complete(&db, server.addr.port()) + .run_until_complete(&db, false, server.addr.port()) .await .json_result() .unwrap(); @@ -90,7 +87,7 @@ async fn test_iteration(db: Pool) -> anyhow::Result<()> { let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }) .arg("items", json!((0..257).collect::>())) - .run_until_complete(&db, server.addr.port()) + .run_until_complete(&db, false, server.addr.port()) .await .json_result() .unwrap(); @@ -141,7 +138,7 @@ async fn test_iteration_parallel(db: Pool) -> anyhow::Result<()> { let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }) .arg("items", json!([])) - .run_until_complete(&db, server.addr.port()) + .run_until_complete(&db, false, server.addr.port()) .await .json_result() .unwrap(); @@ -151,7 +148,7 @@ async fn test_iteration_parallel(db: Pool) -> anyhow::Result<()> { let job = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }) .arg("items", json!((0..50).collect::>())) - .run_until_complete(&db, server.addr.port()) + .run_until_complete(&db, false, server.addr.port()) .await; // println!("{:#?}", job); let result = job.json_result().unwrap(); @@ -167,8 +164,6 @@ async fn test_iteration_parallel(db: Pool) -> anyhow::Result<()> { Ok(()) } - - #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { @@ -214,6 +209,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { continue_on_error: None, skip_if: None, apply_preprocessor: None, + pass_flow_input_directly: None, }, FlowModule { id: "b".to_string(), @@ -258,6 +254,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { continue_on_error: None, skip_if: None, apply_preprocessor: None, + pass_flow_input_directly: None, }], modules_node: None, } @@ -276,6 +273,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { continue_on_error: None, skip_if: None, apply_preprocessor: None, + pass_flow_input_directly: None, }, ], same_worker: false, @@ -288,7 +286,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { for i in 0..50 { println!("deno flow iteration: {}", i); - let job = run_job_in_new_worker_until_complete(&db, job.clone(), port).await; + let job = run_job_in_new_worker_until_complete(&db, false, job.clone(), port).await; // println!("job: {:#?}", job.flow_status); let result = job.json_result().unwrap(); assert_eq!(result, serde_json::json!([2, 4, 6]), "iteration: {}", i); @@ -327,7 +325,7 @@ async fn test_identity(db: Pool) -> anyhow::Result<()> { let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }) - .run_until_complete(&db, server.addr.port()) + .run_until_complete(&db, false, server.addr.port()) .await .json_result() .unwrap(); @@ -335,13 +333,9 @@ async fn test_identity(db: Pool) -> anyhow::Result<()> { Ok(()) } -use windmill_common::flows::FlowModule; -use windmill_common::flows::FlowModuleValue; - #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { - initialize_tracing().await; let server = ApiServer::start(db.clone()).await?; @@ -394,6 +388,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { continue_on_error: None, skip_if: None, apply_preprocessor: None, + pass_flow_input_directly: None, }, FlowModule { id: "b".to_string(), @@ -448,6 +443,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { continue_on_error: None, skip_if: None, apply_preprocessor: None, + pass_flow_input_directly: None, }, FlowModule { id: "e".to_string(), @@ -489,6 +485,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { continue_on_error: None, skip_if: None, apply_preprocessor: None, + pass_flow_input_directly: None, }, ], modules_node: None, @@ -507,6 +504,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { continue_on_error: None, skip_if: None, apply_preprocessor: None, + pass_flow_input_directly: None, }, FlowModule { id: "c".to_string(), @@ -554,6 +552,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { continue_on_error: None, skip_if: None, apply_preprocessor: None, + pass_flow_input_directly: None, }, ], same_worker: true, @@ -562,7 +561,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; - let result = run_job_in_new_worker_until_complete(&db, job.clone(), server.addr.port()) + let result = run_job_in_new_worker_until_complete(&db, false, job.clone(), server.addr.port()) .await .json_result() .unwrap(); @@ -617,7 +616,7 @@ async fn test_flow_result_by_id(db: Pool) -> anyhow::Result<()> { .unwrap(); let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; - let result = run_job_in_new_worker_until_complete(&db, job.clone(), port) + let result = run_job_in_new_worker_until_complete(&db, false, job.clone(), port) .await .json_result() .unwrap(); @@ -664,7 +663,7 @@ async fn test_stop_after_if(db: Pool) -> anyhow::Result<()> { let result = RunJob::from(job.clone()) .arg("n", json!(123)) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -672,7 +671,7 @@ async fn test_stop_after_if(db: Pool) -> anyhow::Result<()> { let cjob = RunJob::from(job.clone()) .arg("n", json!(-123)) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await; let result = cjob.json_result().unwrap(); @@ -725,7 +724,7 @@ async fn test_stop_after_if_nested(db: Pool) -> anyhow::Result<()> { let result = RunJob::from(job.clone()) .arg("n", json!(123)) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -733,7 +732,7 @@ async fn test_stop_after_if_nested(db: Pool) -> anyhow::Result<()> { let cjob = RunJob::from(job.clone()) .arg("n", json!(-123)) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await; let result = cjob.json_result().unwrap(); @@ -788,6 +787,7 @@ async fn test_python_flow(db: Pool) -> anyhow::Result<()> { println!("python flow iteration: {}", i); let result = run_job_in_new_worker_until_complete( &db, + false, JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }, port, ) @@ -825,6 +825,7 @@ async fn test_python_flow_2(db: Pool) -> anyhow::Result<()> { println!("python flow iteration: {}", i); let result = run_job_in_new_worker_until_complete( &db, + false, JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }, port, ) @@ -866,9 +867,11 @@ func main(derp string) (string, error) { concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, + custom_debounce_key: None, + debounce_delay_s: None, })) .arg("derp", json!("world")) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -903,9 +906,11 @@ fn main(world: String) -> Result { concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, + custom_debounce_key: None, + debounce_delay_s: None, })) .arg("world", json!("Hyrule")) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -948,7 +953,7 @@ fn main(world: String) -> Result { // })) // .arg("world", json!("Arakis")) // .arg("b", json!(3)) -// .run_until_complete(&db, port) +// .run_until_complete(&db, false, port) // .await // .json_result() // .unwrap(); @@ -979,9 +984,11 @@ echo "hello $msg" concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, + custom_debounce_key: None, + debounce_delay_s: None, })) .arg("msg", json!("world")) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await; assert_eq!(job.json_result(), Some(json!("hello world"))); Ok(()) @@ -1012,9 +1019,11 @@ def main [ msg: string ] { concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, + custom_debounce_key: None, + debounce_delay_s: None, })) .arg("msg", json!("world")) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await; assert_eq!(job.json_result(), Some(json!("hello world"))); Ok(()) @@ -1065,6 +1074,8 @@ def main [ concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, + custom_debounce_key: None, + debounce_delay_s: None, })) .arg("a", json!("3")) .arg("b", json!("null")) @@ -1083,7 +1094,7 @@ def main [ ]), ) .arg("n", json!("baz")) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -1127,19 +1138,19 @@ public class Main { concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, + custom_debounce_key: None, + debounce_delay_s: None, })) .arg("a", json!(3)) .arg("b", json!(3.0)) .arg("age", json!(30)) .arg("d", json!(3.0)) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await; assert_eq!(job.json_result(), Some(json!("hello world"))); Ok(()) } - - #[sqlx::test(fixtures("base"))] async fn test_bun_job_datetime(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -1164,9 +1175,11 @@ export async function main(a: Date) { concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, + custom_debounce_key: None, + debounce_delay_s: None, })) .arg("a", json!("2024-09-24T10:00:00.000Z")) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -1199,9 +1212,11 @@ export async function main(a: Date) { concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, + custom_debounce_key: None, + debounce_delay_s: None, })) .arg("a", json!("2024-09-24T10:00:00.000Z")) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -1235,10 +1250,12 @@ def main(a: datetime, b: bytes): concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, + custom_debounce_key: None, + debounce_delay_s: None, })) .arg("a", json!("2024-09-24T10:00:00.000Z")) .arg("b", json!("dGVzdA==")) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -1296,7 +1313,7 @@ async fn test_empty_loop_1(db: Pool) -> anyhow::Result<()> { .unwrap(); let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, port) + let result = run_job_in_new_worker_until_complete(&db, false, flow, port) .await .json_result() .unwrap(); @@ -1337,7 +1354,7 @@ async fn test_invalid_first_step(db: Pool) -> anyhow::Result<()> { .unwrap(); let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; - let job = run_job_in_new_worker_until_complete(&db, flow, port).await; + let job = run_job_in_new_worker_until_complete(&db, false, flow, port).await; assert!( serde_json::to_string(&job.json_result().unwrap()).unwrap().contains("Expected an array value in the iterator expression, found: invalid type: map, expected a sequence at line 1 column 0") @@ -1379,7 +1396,7 @@ async fn test_empty_loop_2(db: Pool) -> anyhow::Result<()> { .unwrap(); let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, port) + let result = run_job_in_new_worker_until_complete(&db, false, flow, port) .await .json_result() .unwrap(); @@ -1436,7 +1453,7 @@ async fn test_step_after_loop(db: Pool) -> anyhow::Result<()> { .unwrap(); let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, port) + let result = run_job_in_new_worker_until_complete(&db, false, flow, port) .await .json_result() .unwrap(); @@ -1505,7 +1522,7 @@ async fn test_branchone_simple(db: Pool) -> anyhow::Result<()> { .unwrap(); let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, port) + let result = run_job_in_new_worker_until_complete(&db, false, flow, port) .await .json_result() .unwrap(); @@ -1543,7 +1560,7 @@ async fn test_branchone_with_cond(db: Pool) -> anyhow::Result<()> { .unwrap(); let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, port) + let result = run_job_in_new_worker_until_complete(&db, false, flow, port) .await .json_result() .unwrap(); @@ -1583,7 +1600,7 @@ async fn test_branchall_sequential(db: Pool) -> anyhow::Result<()> { .unwrap(); let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, port) + let result = run_job_in_new_worker_until_complete(&db, false, flow, port) .await .json_result() .unwrap(); @@ -1622,7 +1639,7 @@ async fn test_branchall_simple(db: Pool) -> anyhow::Result<()> { .unwrap(); let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, port) + let result = run_job_in_new_worker_until_complete(&db, false, flow, port) .await .json_result() .unwrap(); @@ -1670,7 +1687,7 @@ async fn test_branchall_skip_failure(db: Pool) -> anyhow::Result<()> { .unwrap(); let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, port) + let result = run_job_in_new_worker_until_complete(&db, false, flow, port) .await .json_result() .unwrap(); @@ -1707,7 +1724,7 @@ async fn test_branchall_skip_failure(db: Pool) -> anyhow::Result<()> { .unwrap(); let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, port) + let result = run_job_in_new_worker_until_complete(&db, false, flow, port) .await .json_result() .unwrap(); @@ -1773,7 +1790,7 @@ async fn test_branchone_nested(db: Pool) -> anyhow::Result<()> { .unwrap(); let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, port) + let result = run_job_in_new_worker_until_complete(&db, false, flow, port) .await .json_result() .unwrap(); @@ -1831,7 +1848,7 @@ async fn test_branchall_nested(db: Pool) -> anyhow::Result<()> { .unwrap(); let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, port) + let result = run_job_in_new_worker_until_complete(&db, false, flow, port) .await .json_result() .unwrap(); @@ -1899,7 +1916,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }) .arg("n", json!(0)) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -1916,7 +1933,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }) .arg("n", json!(1)) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -1933,7 +1950,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }) .arg("n", json!(2)) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -1950,7 +1967,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }) .arg("n", json!(3)) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await .json_result() .unwrap(); @@ -2084,14 +2101,14 @@ async fn test_flow_lock_all(db: Pool) -> anyhow::Result<()> { language: windmill_api_client::types::RawScriptLanguage::Bash, lock: Some(ref lock), .. - }) if lock == "") + }) if lock.is_empty()) || matches!( m.value, windmill_api_client::types::FlowModuleValue::RawScript(RawScript{ language: windmill_api_client::types::RawScriptLanguage::Go | windmill_api_client::types::RawScriptLanguage::Python3 | windmill_api_client::types::RawScriptLanguage::Deno, lock: Some(ref lock), .. - }) if lock.len() > 0), + }) if !lock.is_empty()), "{:?}", m.value ); }); @@ -2270,7 +2287,7 @@ async fn test_complex_flow_restart(db: Pool) -> anyhow::Result<()> { let first_run_result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await; let restarted_flow_result = RunJob::from(JobPayload::RawFlow { @@ -2282,7 +2299,7 @@ async fn test_complex_flow_restart(db: Pool) -> anyhow::Result<()> { branch_or_iteration_n: None, }), }) - .run_until_complete(&db, port) + .run_until_complete(&db, false, port) .await; let first_run_result_int = @@ -2311,7 +2328,7 @@ async fn test_rust_client(db: Pool) -> anyhow::Result<()> { Ok(()) } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", feature = "private"))] #[sqlx::test(fixtures("base", "schedule"))] async fn test_script_schedule_handlers(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -2378,7 +2395,7 @@ async fn test_script_schedule_handlers(db: Pool) -> anyhow::Result<()> let uuid = uuid.unwrap().unwrap(); let completed_job = sqlx::query!( - "SELECT script_path FROM v2_as_completed_job WHERE id = $1", + "SELECT runnable_path as script_path FROM v2_job WHERE id = $1", uuid ) .fetch_one(&db2) @@ -2449,7 +2466,7 @@ async fn test_script_schedule_handlers(db: Pool) -> anyhow::Result<()> let uuid = uuid.unwrap().unwrap(); let completed_job = - sqlx::query!("SELECT script_path FROM v2_as_completed_job WHERE id = $1", uuid) + sqlx::query!("SELECT runnable_path as script_path FROM v2_job WHERE id = $1", uuid) .fetch_one(&db2) .await .unwrap(); @@ -2468,7 +2485,7 @@ async fn test_script_schedule_handlers(db: Pool) -> anyhow::Result<()> Ok(()) } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", feature = "private"))] #[sqlx::test(fixtures("base", "schedule"))] async fn test_flow_schedule_handlers(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -2536,7 +2553,7 @@ async fn test_flow_schedule_handlers(db: Pool) -> anyhow::Result<()> { let uuid = uuid.unwrap().unwrap(); let completed_job = sqlx::query!( - "SELECT script_path FROM v2_as_completed_job WHERE id = $1", + "SELECT runnable_path as script_path FROM v2_job WHERE id = $1", uuid ) .fetch_one(&db2) @@ -2608,7 +2625,7 @@ async fn test_flow_schedule_handlers(db: Pool) -> anyhow::Result<()> { let uuid = uuid.unwrap().unwrap(); let completed_job = - sqlx::query!("SELECT script_path FROM v2_as_completed_job WHERE id = $1", uuid) + sqlx::query!("SELECT runnable_path as script_path FROM v2_job WHERE id = $1", uuid) .fetch_one(&db2) .await .unwrap(); @@ -2627,7 +2644,6 @@ async fn test_flow_schedule_handlers(db: Pool) -> anyhow::Result<()> { Ok(()) } - #[sqlx::test(fixtures("base", "relative_bun"))] async fn test_relative_imports_bun(db: Pool) -> anyhow::Result<()> { let content = r#" @@ -2699,8 +2715,6 @@ export async function main() { Ok(()) } - - #[sqlx::test(fixtures("base", "result_format"))] async fn test_result_format(db: Pool) -> anyhow::Result<()> { let ordered_result_job_id = "1eecb96a-c8b0-4a3d-b1b6-087878c55e41"; @@ -2753,7 +2767,7 @@ async fn test_result_format(db: Pool) -> anyhow::Result<()> { assert_eq!(job_result.get(), correct_result); let response = windmill_api::jobs::run_wait_result( - &db.into(), + &db, Uuid::parse_str(ordered_result_job_id).unwrap(), "test-workspace".to_string(), None, @@ -2764,8 +2778,7 @@ async fn test_result_format(db: Pool) -> anyhow::Result<()> { let result: Box = serde_json::from_slice( &axum::body::to_bytes(response.into_body(), usize::MAX) .await - .unwrap() - .to_vec(), + .unwrap(), ) .unwrap(); assert_eq!(result.get(), correct_result); @@ -2809,7 +2822,7 @@ async fn test_job_labels(db: Pool) -> anyhow::Result<()> { restarted_from: None, }) .arg("world", json!("you")) - .run_until_complete_with(&db, port, |id| async move { + .run_until_complete_with(db, false, port, |id| async move { sqlx::query!( "UPDATE v2_job SET labels = $2 WHERE id = $1 AND $2::TEXT[] IS NOT NULL", id, @@ -2856,13 +2869,15 @@ def heavy_compute(n: int): def send_result(res: int, email: str): print(f"Sending result {res} to {email}") return "OK" - + def main(n: int): l = [] for i in range(n): l.append(heavy_compute(i)) print(l) return [send_result(sum(l), "example@example.com"), n] + + "#; #[cfg(feature = "python")] @@ -2875,7 +2890,7 @@ async fn test_workflow_as_code(db: Pool) -> anyhow::Result<()> { // workflow as code require at least 2 workers: let db = &db; in_test_worker( - &db, + db, async move { let job = RunJob::from(JobPayload::Code(RawCode { language: ScriptLang::Python3, @@ -2883,7 +2898,7 @@ async fn test_workflow_as_code(db: Pool) -> anyhow::Result<()> { ..RawCode::default() })) .arg("n", json!(3)) - .run_until_complete(&db, port) + .run_until_complete(db, false, port) .await; assert_eq!(job.json_result().unwrap(), json!(["OK", 3])); @@ -2929,4 +2944,3 @@ async fn test_workflow_as_code(db: Pool) -> anyhow::Result<()> { .await; Ok(()) } - diff --git a/backend/update_sqlx.sh b/backend/update_sqlx.sh index d8bc473f29..5425706876 100755 --- a/backend/update_sqlx.sh +++ b/backend/update_sqlx.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash -# Default directory -EE_DIR="../windmill-ee-private" +set -e # Parse arguments while [[ "$#" -gt 0 ]]; do @@ -12,8 +11,6 @@ while [[ "$#" -gt 0 ]]; do shift done -./substitute_ee_code.sh --dir "$EE_DIR" - # Check if running on macOS if [[ "$(uname)" == "Darwin" ]]; then echo "Running on macOS - substituting samael..." @@ -21,9 +18,17 @@ if [[ "$(uname)" == "Darwin" ]]; then sed -i '' 's/^samael = { version="0.0.14", features = \["xmlsec"\] }/#samael = { version="0.0.14", features = ["xmlsec"] }/' Cargo.toml # Uncomment the git-based samael dependency sed -i '' 's/^# \(samael = { git="https:\/\/github.com\/njaremko\/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = \["xmlsec"\] }\)/\1/' Cargo.toml + + # Run cargo sqlx prepare with deno_core_mac + echo "Running cargo sqlx prepare with deno_core_mac..." + cargo sqlx prepare --workspace -- --all-targets --features all_sqlx_features,private,deno_core_mac +else + # Run cargo sqlx prepare + echo "Running cargo sqlx prepare..." + cargo sqlx prepare --workspace -- --all-targets --features all_sqlx_features,private fi -cargo sqlx prepare --workspace -- --all-targets --all-features + # Undo the samael changes on macOS if [[ "$(uname)" == "Darwin" ]]; then diff --git a/backend/windmill-api-client/src/codegen.rs b/backend/windmill-api-client/src/codegen.rs index 2ef8fe8cac..a295d9df8f 100644 --- a/backend/windmill-api-client/src/codegen.rs +++ b/backend/windmill-api-client/src/codegen.rs @@ -8,9 +8,80 @@ pub mod types { #[allow(unused_imports)] use std::convert::TryFrom; #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct AiAgent { + pub input_transforms: std::collections::HashMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parallel: Option, + pub tools: Vec, + #[serde(rename = "type")] + pub type_: AiAgentType, + } + impl From<&AiAgent> for AiAgent { + fn from(value: &AiAgent) -> Self { + value.clone() + } + } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum AiAgentType { + #[serde(rename = "aiagent")] + Aiagent, + } + impl From<&AiAgentType> for AiAgentType { + fn from(value: &AiAgentType) -> Self { + value.clone() + } + } + impl ToString for AiAgentType { + fn to_string(&self) -> String { + match *self { + Self::Aiagent => "aiagent".to_string(), + } + } + } + impl std::str::FromStr for AiAgentType { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "aiagent" => Ok(Self::Aiagent), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for AiAgentType { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for AiAgentType { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for AiAgentType { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct AiConfig { #[serde(default, skip_serializing_if = "Option::is_none")] pub code_completion_model: Option, + #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] + pub custom_prompts: std::collections::HashMap, #[serde(default, skip_serializing_if = "Option::is_none")] pub default_model: Option, #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] @@ -1295,8 +1366,8 @@ pub mod types { Websocket, #[serde(rename = "kafka")] Kafka, - #[serde(rename = "email")] - Email, + #[serde(rename = "default_email")] + DefaultEmail, #[serde(rename = "nats")] Nats, #[serde(rename = "postgres")] @@ -1307,6 +1378,8 @@ pub mod types { Mqtt, #[serde(rename = "gcp")] Gcp, + #[serde(rename = "email")] + Email, } impl From<&CaptureTriggerKind> for CaptureTriggerKind { fn from(value: &CaptureTriggerKind) -> Self { @@ -1320,12 +1393,13 @@ pub mod types { Self::Http => "http".to_string(), Self::Websocket => "websocket".to_string(), Self::Kafka => "kafka".to_string(), - Self::Email => "email".to_string(), + Self::DefaultEmail => "default_email".to_string(), Self::Nats => "nats".to_string(), Self::Postgres => "postgres".to_string(), Self::Sqs => "sqs".to_string(), Self::Mqtt => "mqtt".to_string(), Self::Gcp => "gcp".to_string(), + Self::Email => "email".to_string(), } } } @@ -1337,12 +1411,13 @@ pub mod types { "http" => Ok(Self::Http), "websocket" => Ok(Self::Websocket), "kafka" => Ok(Self::Kafka), - "email" => Ok(Self::Email), + "default_email" => Ok(Self::DefaultEmail), "nats" => Ok(Self::Nats), "postgres" => Ok(Self::Postgres), "sqs" => Ok(Self::Sqs), "mqtt" => Ok(Self::Mqtt), "gcp" => Ok(Self::Gcp), + "email" => Ok(Self::Email), _ => Err("invalid value"), } } @@ -1493,6 +1568,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK Flownode, #[serde(rename = "appscript")] Appscript, + #[serde(rename = "aiagent")] + Aiagent, } impl From<&CompletedJobJobKind> for CompletedJobJobKind { fn from(value: &CompletedJobJobKind) -> Self { @@ -1516,6 +1593,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK Self::Flowscript => "flowscript".to_string(), Self::Flownode => "flownode".to_string(), Self::Appscript => "appscript".to_string(), + Self::Aiagent => "aiagent".to_string(), } } } @@ -1537,6 +1615,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK "flowscript" => Ok(Self::Flowscript), "flownode" => Ok(Self::Flownode), "appscript" => Ok(Self::Appscript), + "aiagent" => Ok(Self::Aiagent), _ => Err("invalid value"), } } @@ -1705,6 +1784,21 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct CreateWorkspaceFork { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub color: Option, + pub id: String, + pub name: String, + pub parent_workspace_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub username: Option, + } + impl From<&CreateWorkspaceFork> for CreateWorkspaceFork { + fn from(value: &CreateWorkspaceFork) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct CriticalAlert { ///Acknowledgment status of the alert, can be true, false, or null if not set #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1799,6 +1893,24 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct DependencyMap { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub imported_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub importer_kind: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub importer_node_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub importer_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace_id: Option, + } + impl From<&DependencyMap> for DependencyMap { + fn from(value: &DependencyMap) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct DucklakeSettings { pub ducklakes: std::collections::HashMap, } @@ -1909,6 +2021,27 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct EditEmailTrigger { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + pub is_flow: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub local_part: Option, + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, + pub script_path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspaced_local_part: Option, + } + impl From<&EditEmailTrigger> for EditEmailTrigger { + fn from(value: &EditEmailTrigger) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct EditHttpTrigger { pub authentication_method: AuthenticationMethod, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -2234,6 +2367,23 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct EmailTrigger { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + pub local_part: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspaced_local_part: Option, + } + impl From<&EmailTrigger> for EmailTrigger { + fn from(value: &EmailTrigger) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct EndpointTool { ///JSON schema for request body #[serde(default, skip_serializing_if = "Option::is_none")] @@ -2481,7 +2631,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Option::is_none")] pub suspend: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub timeout: Option, + pub timeout: Option, pub value: FlowModuleValue, } impl From<&FlowModule> for FlowModule { @@ -2555,6 +2705,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK BranchOne(BranchOne), BranchAll(BranchAll), Identity(Identity), + AiAgent(AiAgent), } impl From<&FlowModuleValue> for FlowModuleValue { fn from(value: &FlowModuleValue) -> Self { @@ -2601,6 +2752,11 @@ the execution of this script will be permissioned_as and by extension its DT_TOK Self::Identity(value) } } + impl From for FlowModuleValue { + fn from(value: AiAgent) -> Self { + Self::AiAgent(value) + } + } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct FlowPreview { pub args: ScriptArgs, @@ -2648,6 +2804,10 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct FlowStatusModule { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub agent_actions: Vec>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub agent_actions_success: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub approvers: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -4810,6 +4970,26 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct NewEmailTrigger { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + pub is_flow: bool, + pub local_part: String, + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, + pub script_path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspaced_local_part: Option, + } + impl From<&NewEmailTrigger> for NewEmailTrigger { + fn from(value: &NewEmailTrigger) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct NewHttpTrigger { pub authentication_method: AuthenticationMethod, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -5907,6 +6087,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Preview { pub args: ScriptArgs, + ///The code to run #[serde(default, skip_serializing_if = "Option::is_none")] pub content: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -5917,8 +6098,10 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub language: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub lock: Option, + ///The path to the script #[serde(default, skip_serializing_if = "Option::is_none")] pub path: Option, + ///The hash of the script #[serde(default, skip_serializing_if = "Option::is_none")] pub script_hash: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -6127,6 +6310,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK Flownode, #[serde(rename = "appscript")] Appscript, + #[serde(rename = "aiagent")] + Aiagent, } impl From<&QueuedJobJobKind> for QueuedJobJobKind { fn from(value: &QueuedJobJobKind) -> Self { @@ -6150,6 +6335,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK Self::Flowscript => "flowscript".to_string(), Self::Flownode => "flownode".to_string(), Self::Appscript => "appscript".to_string(), + Self::Aiagent => "aiagent".to_string(), } } } @@ -6171,6 +6357,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK "flowscript" => Ok(Self::Flowscript), "flownode" => Ok(Self::Flownode), "appscript" => Ok(Self::Appscript), + "aiagent" => Ok(Self::Aiagent), _ => Err("invalid value"), } } @@ -6676,6 +6863,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub constant: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub exponential: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry_if: Option, } impl From<&Retry> for Retry { fn from(value: &Retry) -> Self { @@ -6710,6 +6899,15 @@ the execution of this script will be permissioned_as and by extension its DT_TOK value.clone() } } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct RetryRetryIf { + pub expr: String, + } + impl From<&RetryRetryIf> for RetryRetryIf { + fn from(value: &RetryRetryIf) -> Self { + value.clone() + } + } #[derive( Clone, Copy, @@ -7861,6 +8059,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct TriggersCount { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default_email_count: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub email_count: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -8069,10 +8269,14 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[derive(Clone, Debug, Deserialize, Serialize)] pub struct UserWorkspaceListWorkspacesItem { pub color: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub created_by: Option, pub id: String, pub name: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub operator_settings: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_workspace_id: Option, pub username: String, } impl From<&UserWorkspaceListWorkspacesItem> for UserWorkspaceListWorkspacesItem { @@ -8479,6 +8683,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub id: String, pub name: String, pub owner: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_workspace_id: Option, } impl From<&Workspace> for Workspace { fn from(value: &Workspace) -> Self { @@ -8550,6 +8756,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub email: String, pub is_admin: bool, pub operator: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_workspace_id: Option, pub workspace_id: String, } impl From<&WorkspaceInvite> for WorkspaceInvite { @@ -8561,7 +8769,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[derive(Clone, Debug)] /**Client for Windmill API -Version: 1.526.1*/ +Version: 1.543.0*/ pub struct Client { pub(crate) baseurl: String, pub(crate) client: reqwest::Client, @@ -8607,7 +8815,7 @@ impl Client { /// This string is pulled directly from the source OpenAPI /// document and may be in any format the API selects. pub fn api_version(&self) -> &'static str { - "1.526.1" + "1.543.0" } } impl Client { diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 60d79e3b31..34cb7adfe9 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -40,7 +40,7 @@ mcp = ["dep:rmcp"] python = [] [dependencies] -rmcp = { version = "0.2.1", features=["transport-streamable-http-server", "transport-streamable-http-server-session", "transport-worker"], optional = true } +rmcp = { version = "0.8.1", features=["transport-streamable-http-server", "transport-streamable-http-server-session", "transport-worker"], optional = true } windmill-queue.workspace = true windmill-common = { workspace = true, default-features = false } windmill-audit.workspace = true diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index e864874d22..0fa948348a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.542.1 + version: 1.573.3 title: Windmill API contact: @@ -554,6 +554,31 @@ paths: items: $ref: "#/components/schemas/ExportedUser" + /users/onboarding: + post: + summary: Submit user onboarding data + operationId: submitOnboardingData + tags: + - user + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + touch_point: + type: string + use_case: + type: string + responses: + '200': + description: Onboarding data submitted successfully + content: + application/json: + schema: + type: string + /w/{workspace}/users/delete/{username}: delete: summary: delete user (require admin privilege) @@ -575,6 +600,27 @@ paths: text/plain: schema: type: string + /w/{workspace}/users/convert_to_group/{username}: + post: + summary: convert manual user to group user (require admin privilege) + operationId: convertUserToGroup + tags: + - user + - admin + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: username + in: path + required: true + schema: + type: string + responses: + "200": + description: convert user to group user + content: + text/plain: + schema: + type: string /github_app/connected_repositories: get: @@ -582,6 +628,14 @@ paths: operationId: getGlobalConnectedRepositories tags: - Git Sync + parameters: + - name: page + in: query + description: Page number for pagination (default 1) + required: false + schema: + type: integer + default: 1 responses: "200": description: connected repositories @@ -674,12 +728,14 @@ paths: schema: type: string - /workspaces/create_fork: + /w/{workspace}/workspaces/create_fork: post: summary: create forked workspace operationId: createWorkspaceFork tags: - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" requestBody: description: new forked workspace required: true @@ -749,34 +805,26 @@ paths: schema: type: boolean - /settings/databases_exist: + /settings/get_ducklake_instance_catalog_db_status: post: - summary: checks that all given databases exist or else return the ones that don't - operationId: databasesExist + summary: Returns the set-up statuses of ducklake instance catalog dbs + operationId: getDucklakeInstanceCatalogDbStatus tags: - setting - requestBody: - required: true - content: - application/json: - schema: - type: array - items: - type: string responses: "200": - description: databases that do not exist + description: Statuses of all ducklake instance catalog dbs content: application/json: schema: - type: array - items: - type: string + type: object + additionalProperties: + $ref: "#/components/schemas/DucklakeInstanceCatalogDbStatus" - /settings/create_ducklake_database/{name}: + /settings/setup_ducklake_catalog_db/{name}: post: summary: Runs CREATE DATABASE on the Windmill Postgres and grants access to the ducklake_user - operationId: createDucklakeDatabase + operationId: setupDucklakeCatalogDb tags: - setting parameters: @@ -791,7 +839,8 @@ paths: description: status content: application/json: - schema: {} + schema: + $ref: "#/components/schemas/DucklakeInstanceCatalogDbStatus" /settings/global/{key}: get: @@ -1306,8 +1355,6 @@ paths: type: string endpoint_sync: type: string - endpoint_openai_sync: - type: string summary: type: string description: @@ -1318,7 +1365,6 @@ paths: - workspace - endpoint_async - endpoint_sync - - endpoint_openai_sync - summary - kind @@ -1707,6 +1753,11 @@ paths: - workspace parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: only_delete_forks + in: query + required: false + schema: + type: boolean responses: "200": description: status @@ -1979,6 +2030,10 @@ paths: type: string slack_command_script: type: string + slack_oauth_client_id: + type: string + slack_oauth_client_secret: + type: string teams_team_id: type: string teams_command_script: @@ -2160,6 +2215,40 @@ paths: schema: type: string + /w/{workspace}/workspaces/rebuild_dependency_map: + post: + summary: rebuild dependency map + operationId: rebuildDependencyMap + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/get_dependency_map: + get: + summary: get dependency map + operationId: getDependencyMap + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: dmap + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/DependencyMap" + /w/{workspace}/workspaces/edit_slack_command: post: summary: edit slack command @@ -2187,6 +2276,73 @@ paths: schema: type: string + /w/{workspace}/workspaces/slack_oauth_config: + get: + summary: get workspace slack oauth config + operationId: getWorkspaceSlackOauthConfig + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: slack oauth config + content: + application/json: + schema: + type: object + properties: + slack_oauth_client_id: + type: string + nullable: true + slack_oauth_client_secret: + type: string + nullable: true + description: Masked with *** if set + post: + summary: set workspace slack oauth config + operationId: setWorkspaceSlackOauthConfig + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: Slack OAuth Configuration + required: true + content: + application/json: + schema: + type: object + required: + - slack_oauth_client_id + - slack_oauth_client_secret + properties: + slack_oauth_client_id: + type: string + slack_oauth_client_secret: + type: string + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + delete: + summary: delete workspace slack oauth config + operationId: deleteWorkspaceSlackOauthConfig + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + /w/{workspace}/workspaces/edit_teams_command: post: summary: edit teams command @@ -2221,6 +2377,12 @@ paths: - workspace parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: search + in: query + description: Search teams by name + required: false + schema: + type: string responses: "200": description: status @@ -2238,15 +2400,27 @@ paths: /w/{workspace}/workspaces/available_teams_channels: get: - summary: list available teams channels + summary: list available channels for a specific team operationId: listAvailableTeamsChannels tags: - workspace parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: team_id + in: query + description: Microsoft Teams team ID + required: true + schema: + type: string + - name: search + in: query + description: Search channels by name + required: false + schema: + type: string responses: "200": - description: status + description: List of channels for the specified team content: application/json: schema: @@ -2258,10 +2432,6 @@ paths: type: string channel_id: type: string - service_url: - type: string - tenant_id: - type: string /w/{workspace}/workspaces/connect_teams: post: @@ -3695,6 +3865,7 @@ paths: properties: refresh_token: type: string + description: "OAuth refresh token. For authorization_code flow, this contains the actual refresh token. For client_credentials flow, this must be set to an empty string." expires_in: type: integer client: @@ -3712,6 +3883,7 @@ paths: type: string description: "OAuth token URL override for resource-level authentication (client_credentials flow only)" required: + - refresh_token - expires_in - client responses: @@ -3923,22 +4095,6 @@ paths: items: type: string - /teams/sync: - post: - operationId: syncTeams - summary: synchronize Microsoft Teams information (teams/channels) - tags: - - teams - responses: - "200": - description: Teams information successfully synchronized - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/TeamInfo" - /teams/activities: post: summary: send update to Microsoft Teams activity @@ -4165,6 +4321,33 @@ paths: application/json: schema: {} + /w/{workspace}/resources/git_commit_hash/{path}: + get: + summary: get git repository latest commit hash + operationId: getGitCommitHash + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + - name: git_ssh_identity + in: query + schema: + type: string + responses: + "200": + description: git commit hash + content: + application/json: + schema: + type: object + properties: + commit_hash: + type: string + description: Latest commit hash from git ls-remote + required: + - commit_hash + /w/{workspace}/resources/exists/{path}: get: summary: does resource exists @@ -4243,6 +4426,35 @@ paths: - path - value + /w/{workspace}/resources/mcp_tools/{path}: + get: + summary: get MCP tools from resource + operationId: getMcpTools + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: list of MCP tools + content: + application/json: + schema: + type: array + items: + type: object + properties: + name: + type: string + description: + type: string + parameters: + type: object + required: + - name + - parameters + /w/{workspace}/resources/list_names/{name}: get: summary: list resource names @@ -4683,6 +4895,27 @@ paths: - content - language + /scripts/hub/pick/{path}: + get: + summary: record hub script pick + operationId: pickHubScriptByPath + tags: + - script + parameters: + - $ref: "#/components/parameters/ScriptPath" + responses: + "200": + description: script pick recorded + content: + application/json: + schema: + type: object + properties: + success: + type: boolean + required: + - success + /scripts/hub/top: get: summary: get top hub scripts @@ -5629,35 +5862,6 @@ paths: type: string format: uuid - /w/{workspace}/jobs/openai_sync/p/{path}: - post: - summary: run script by path in openai format - operationId: openaiSyncScriptByPath - tags: - - job - parameters: - - $ref: "#/components/parameters/WorkspaceId" - - $ref: "#/components/parameters/ScriptPath" - - $ref: "#/components/parameters/ParentJob" - - $ref: "#/components/parameters/NewJobId" - - $ref: "#/components/parameters/IncludeHeader" - - $ref: "#/components/parameters/QueueLimit" - - requestBody: - description: script args - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/ScriptArgs" - - responses: - "200": - description: job result - content: - application/json: - schema: {} - /w/{workspace}/jobs/run_wait_result/p/{path}: post: summary: run script by path @@ -5673,6 +5877,7 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/IncludeHeader" - $ref: "#/components/parameters/QueueLimit" + - $ref: "#/components/parameters/SkipPreprocessor" requestBody: description: script args @@ -5704,34 +5909,7 @@ paths: - $ref: "#/components/parameters/IncludeHeader" - $ref: "#/components/parameters/QueueLimit" - $ref: "#/components/parameters/Payload" - - responses: - "200": - description: job result - content: - application/json: - schema: {} - - /w/{workspace}/jobs/openai_sync/f/{path}: - post: - summary: run flow by path and wait until completion in openai format - operationId: openaiSyncFlowByPath - tags: - - job - parameters: - - $ref: "#/components/parameters/WorkspaceId" - - $ref: "#/components/parameters/ScriptPath" - - $ref: "#/components/parameters/IncludeHeader" - - $ref: "#/components/parameters/QueueLimit" - - $ref: "#/components/parameters/NewJobId" - - requestBody: - description: script args - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/ScriptArgs" + - $ref: "#/components/parameters/SkipPreprocessor" responses: "200": @@ -5752,6 +5930,13 @@ paths: - $ref: "#/components/parameters/IncludeHeader" - $ref: "#/components/parameters/QueueLimit" - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/SkipPreprocessor" + - name: memory_id + description: memory ID for chat-enabled flows + in: query + schema: + type: string + format: uuid requestBody: description: script args @@ -5768,6 +5953,230 @@ paths: application/json: schema: {} + /w/{workspace}/jobs/run_and_stream/f/{path}: + post: + summary: run flow by path and stream updates via SSE + operationId: runAndStreamFlowByPath + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/IncludeHeader" + - $ref: "#/components/parameters/QueueLimit" + - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/SkipPreprocessor" + - name: memory_id + description: memory ID for chat-enabled flows + in: query + schema: + type: string + format: uuid + - name: poll_delay_ms + description: delay between polling for job updates in milliseconds + in: query + schema: + type: integer + format: int64 + + requestBody: + description: flow args + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptArgs" + + responses: + "200": + description: server-sent events stream of job updates + content: + text/event-stream: + schema: + type: string + + get: + summary: run flow by path with GET and stream updates via SSE + operationId: runAndStreamFlowByPathGet + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/IncludeHeader" + - $ref: "#/components/parameters/QueueLimit" + - $ref: "#/components/parameters/Payload" + - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/SkipPreprocessor" + - name: memory_id + description: memory ID for chat-enabled flows + in: query + schema: + type: string + format: uuid + - name: poll_delay_ms + description: delay between polling for job updates in milliseconds + in: query + schema: + type: integer + format: int64 + + responses: + "200": + description: server-sent events stream of job updates + content: + text/event-stream: + schema: + type: string + + /w/{workspace}/jobs/run_and_stream/p/{path}: + post: + summary: run script by path and stream updates via SSE + operationId: runAndStreamScriptByPath + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/ParentJob" + - $ref: "#/components/parameters/WorkerTag" + - $ref: "#/components/parameters/CacheTtl" + - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/IncludeHeader" + - $ref: "#/components/parameters/QueueLimit" + - $ref: "#/components/parameters/SkipPreprocessor" + - name: poll_delay_ms + description: delay between polling for job updates in milliseconds + in: query + schema: + type: integer + format: int64 + + requestBody: + description: script args + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptArgs" + + responses: + "200": + description: server-sent events stream of job updates + content: + text/event-stream: + schema: + type: string + + get: + summary: run script by path with GET and stream updates via SSE + operationId: runAndStreamScriptByPathGet + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/ParentJob" + - $ref: "#/components/parameters/WorkerTag" + - $ref: "#/components/parameters/CacheTtl" + - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/IncludeHeader" + - $ref: "#/components/parameters/QueueLimit" + - $ref: "#/components/parameters/Payload" + - $ref: "#/components/parameters/SkipPreprocessor" + - name: poll_delay_ms + description: delay between polling for job updates in milliseconds + in: query + schema: + type: integer + format: int64 + + responses: + "200": + description: server-sent events stream of job updates + content: + text/event-stream: + schema: + type: string + + /w/{workspace}/jobs/run_and_stream/h/{hash}: + post: + summary: run script by hash and stream updates via SSE + operationId: runAndStreamScriptByHash + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: hash + in: path + required: true + schema: + type: string + - $ref: "#/components/parameters/ParentJob" + - $ref: "#/components/parameters/WorkerTag" + - $ref: "#/components/parameters/CacheTtl" + - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/IncludeHeader" + - $ref: "#/components/parameters/QueueLimit" + - $ref: "#/components/parameters/SkipPreprocessor" + - name: poll_delay_ms + description: delay between polling for job updates in milliseconds + in: query + schema: + type: integer + format: int64 + + requestBody: + description: script args + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptArgs" + + responses: + "200": + description: server-sent events stream of job updates + content: + text/event-stream: + schema: + type: string + + get: + summary: run script by hash with GET and stream updates via SSE + operationId: runAndStreamScriptByHashGet + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: hash + in: path + required: true + schema: + type: string + - $ref: "#/components/parameters/ParentJob" + - $ref: "#/components/parameters/WorkerTag" + - $ref: "#/components/parameters/CacheTtl" + - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/IncludeHeader" + - $ref: "#/components/parameters/QueueLimit" + - $ref: "#/components/parameters/Payload" + - $ref: "#/components/parameters/SkipPreprocessor" + - name: poll_delay_ms + description: delay between polling for job updates in milliseconds + in: query + schema: + type: integer + format: int64 + + responses: + "200": + description: server-sent events stream of job updates + content: + text/event-stream: + schema: + type: string + /w/{workspace}/jobs/result_by_id/{flow_job_id}/{node_id}: get: summary: get job result by id @@ -6275,6 +6684,88 @@ paths: schema: type: string + /w/{workspace}/flow_conversations/list: + get: + summary: list flow conversations + operationId: listFlowConversations + tags: + - flow_conversation + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" + - name: flow_path + description: filter conversations by flow path + in: query + schema: + type: string + responses: + "200": + description: flow conversations list + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/FlowConversation" + + /w/{workspace}/flow_conversations/delete/{conversation_id}: + delete: + summary: delete flow conversation + operationId: deleteFlowConversation + tags: + - flow_conversation + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: conversation_id + description: conversation id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: flow conversation deleted + content: + text/plain: + schema: + type: string + + /w/{workspace}/flow_conversations/{conversation_id}/messages: + get: + summary: list conversation messages + operationId: listConversationMessages + tags: + - flow_conversation + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" + - name: conversation_id + description: conversation id + in: path + required: true + schema: + type: string + format: uuid + - name: after_id + description: id to fetch only the messages after that id + in: query + required: false + schema: + type: string + format: uuid + responses: + "200": + description: conversation messages + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/FlowConversationMessage" + /w/{workspace}/raw_apps/list: get: summary: list all raw apps @@ -6723,6 +7214,23 @@ paths: schema: type: string + /w/{workspace}/apps/secret_of_latest_version/{path}: + get: + summary: get public secret of latest version of an app bundle + operationId: getPublicSecretOfLatestVersionOfApp + tags: + - app + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: app secret + content: + text/plain: + schema: + type: string + /w/{workspace}/apps/get/v/{id}: get: summary: get app by version @@ -7137,11 +7645,7 @@ paths: in: query schema: type: integer - - name: skip_preprocessor - description: skip the preprocessor - in: query - schema: - type: boolean + - $ref: "#/components/parameters/SkipPreprocessor" - $ref: "#/components/parameters/ParentJob" - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/NewJobId" @@ -7151,6 +7655,12 @@ paths: in: query schema: type: boolean + - name: memory_id + description: memory ID for chat-enabled flows + in: query + schema: + type: string + format: uuid requestBody: description: flow args required: true @@ -7299,11 +7809,7 @@ paths: in: query schema: type: integer - - name: skip_preprocessor - description: skip the preprocessor - in: query - schema: - type: boolean + - $ref: "#/components/parameters/SkipPreprocessor" - $ref: "#/components/parameters/ParentJob" - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/CacheTtl" @@ -7527,6 +8033,30 @@ paths: application/json: schema: {} + /w/{workspace}/jobs/run/dynamic_select: + post: + summary: run dynamic select helper function + operationId: runDynamicSelect + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: dynamic select request + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/DynamicInputData" + responses: + "201": + description: dynamic select job created + content: + text/plain: + schema: + type: string + format: uuid + /w/{workspace}/jobs/queue/list: get: summary: list all queued jobs @@ -7678,11 +8208,12 @@ paths: - $ref: "#/components/parameters/StartedAfter" - $ref: "#/components/parameters/CreatedBefore" - $ref: "#/components/parameters/CreatedAfter" - - $ref: "#/components/parameters/CreatedOrStartedBefore" + - $ref: "#/components/parameters/CompletedBefore" + - $ref: "#/components/parameters/CompletedAfter" + - $ref: "#/components/parameters/CreatedBeforeQueue" + - $ref: "#/components/parameters/CreatedAfterQueue" - $ref: "#/components/parameters/Running" - $ref: "#/components/parameters/ScheduledForBeforeNow" - - $ref: "#/components/parameters/CreatedOrStartedAfter" - - $ref: "#/components/parameters/CreatedOrStartedAfterCompletedJob" - $ref: "#/components/parameters/JobKinds" - $ref: "#/components/parameters/Suspended" - $ref: "#/components/parameters/ArgsFilter" @@ -7791,6 +8322,10 @@ paths: - job parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: force_cancel + in: query + schema: + type: boolean requestBody: description: uuids of the jobs to cancel required: true @@ -7888,18 +8423,18 @@ paths: - $ref: "#/components/parameters/StartedAfter" - $ref: "#/components/parameters/CreatedBefore" - $ref: "#/components/parameters/CreatedAfter" - - $ref: "#/components/parameters/CreatedOrStartedBefore" + - $ref: "#/components/parameters/CompletedBefore" + - $ref: "#/components/parameters/CompletedAfter" + - $ref: "#/components/parameters/CreatedBeforeQueue" + - $ref: "#/components/parameters/CreatedAfterQueue" - $ref: "#/components/parameters/Running" - $ref: "#/components/parameters/ScheduledForBeforeNow" - - $ref: "#/components/parameters/CreatedOrStartedAfter" - - $ref: "#/components/parameters/CreatedOrStartedAfterCompletedJob" - $ref: "#/components/parameters/JobKinds" - $ref: "#/components/parameters/Suspended" - $ref: "#/components/parameters/ArgsFilter" - $ref: "#/components/parameters/Tag" - $ref: "#/components/parameters/ResultFilter" - $ref: "#/components/parameters/AllowWildcards" - - $ref: "#/components/parameters/Page" - $ref: "#/components/parameters/PerPage" - name: is_skipped description: is the job skipped @@ -8088,6 +8623,34 @@ paths: application/json: schema: {} + /w/{workspace}/jobs_u/queue/get_started_at_by_ids: + post: + summary: get started at by ids + operationId: getStartedAtByIds + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: ids + required: true + content: + application/json: + schema: + type: array + items: + type: string + responses: + "200": + description: started at by ids + content: + application/json: + schema: + type: array + items: + type: string + format: date-time + /w/{workspace}/jobs_u/getupdate/{id}: get: summary: get job updates @@ -12054,6 +12617,8 @@ paths: required: true schema: type: string + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" responses: "200": description: List of autoscaling events @@ -13083,6 +13648,160 @@ paths: schema: $ref: "#/components/schemas/WindmillFilePreview" + /w/{workspace}/job_helpers/list_git_repo_files: + get: + summary: List the file keys available in instance object storage with resource-based access control + operationId: listGitRepoFiles + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: max_keys + in: query + required: true + schema: + type: integer + - name: marker + in: query + schema: + type: string + - name: prefix + in: query + required: false + schema: + type: string + description: Must follow format gitrepos/{workspace_id}/{resource_path}/... + - name: storage + in: query + schema: + type: string + responses: + "200": + description: List of file keys + content: + application/json: + schema: + type: object + properties: + next_marker: + type: string + windmill_large_files: + type: array + items: + $ref: "#/components/schemas/WindmillLargeFile" + restricted_access: + type: boolean + required: + - windmill_large_files + + /w/{workspace}/job_helpers/load_git_repo_file_preview: + get: + summary: Load a preview of a file from instance storage with resource-based access control + operationId: loadGitRepoFilePreview + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: file_key + in: query + required: true + schema: + type: string + description: Must follow format gitrepos/{workspace_id}/{resource_path}/... + - name: file_size_in_bytes + in: query + schema: + type: integer + - name: file_mime_type + in: query + schema: + type: string + - name: csv_separator + in: query + schema: + type: string + - name: csv_has_header + in: query + schema: + type: boolean + - name: read_bytes_from + in: query + schema: + type: integer + - name: read_bytes_length + in: query + schema: + type: integer + - name: storage + in: query + schema: + type: string + responses: + "200": + description: FilePreview + content: + application/json: + schema: + $ref: "#/components/schemas/WindmillFilePreview" + + /w/{workspace}/job_helpers/load_git_repo_file_metadata: + get: + summary: Load file metadata from instance storage with resource-based access control + operationId: loadGitRepoFileMetadata + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: file_key + in: query + required: true + schema: + type: string + description: Must follow format gitrepos/{workspace_id}/{resource_path}/... + - name: storage + in: query + schema: + type: string + responses: + "200": + description: FileMetadata + content: + application/json: + schema: + $ref: "#/components/schemas/WindmillFileMetadata" + + /w/{workspace}/job_helpers/check_s3_folder_exists: + get: + summary: Check if S3 path exists and is a folder + operationId: checkS3FolderExists + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: file_key + description: S3 file key to check (e.g., gitrepos/{workspace_id}/u/user/resource/{commit_hash}) + in: query + required: true + schema: + type: string + responses: + "200": + description: S3 folder existence check result + content: + application/json: + schema: + type: object + properties: + exists: + type: boolean + description: Whether the path exists + is_folder: + type: boolean + description: Whether the path is a folder (true) or file (false) + required: + - exists + - is_folder + /w/{workspace}/job_helpers/load_parquet_preview/{path}: get: summary: Load a preview of a parquet file @@ -13319,6 +14038,67 @@ paths: required: - file_key + /w/{workspace}/job_helpers/upload_git_repo_file_to_instance_storage: + post: + summary: Upload a file to the instance storage gitrepos section for viewing + operationId: gitRepoViewerFileUpload + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: file_key + in: query + required: false + schema: + type: string + - name: file_extension + in: query + required: false + schema: + type: string + - name: s3_resource_path + in: query + required: false + schema: + type: string + - name: resource_type + in: query + required: false + schema: + type: string + - name: storage + in: query + schema: + type: string + - name: content_type + in: query + schema: + type: string + - name: content_disposition + in: query + schema: + type: string + requestBody: + description: File content + required: true + content: + application/octet-stream: + schema: + type: string + format: binary + responses: + "200": + description: File upload status + content: + application/json: + schema: + type: object + properties: + file_key: + type: string + required: + - file_key + /w/{workspace}/job_helpers/download_s3_file: get: summary: Download file from S3 bucket @@ -13616,11 +14396,12 @@ paths: - $ref: "#/components/parameters/ScriptExactHash" - $ref: "#/components/parameters/StartedBefore" - $ref: "#/components/parameters/StartedAfter" - - $ref: "#/components/parameters/CreatedOrStartedBefore" - $ref: "#/components/parameters/Running" - $ref: "#/components/parameters/ScheduledForBeforeNow" - - $ref: "#/components/parameters/CreatedOrStartedAfter" - - $ref: "#/components/parameters/CreatedOrStartedAfterCompletedJob" + - $ref: "#/components/parameters/CompletedBefore" + - $ref: "#/components/parameters/CompletedAfter" + - $ref: "#/components/parameters/CreatedBeforeQueue" + - $ref: "#/components/parameters/CreatedAfterQueue" - $ref: "#/components/parameters/JobKinds" - $ref: "#/components/parameters/ArgsFilter" - $ref: "#/components/parameters/Tag" @@ -14136,6 +14917,12 @@ components: in: query schema: type: string + SkipPreprocessor: + name: skip_preprocessor + description: skip the preprocessor + in: query + schema: + type: boolean Payload: name: payload description: | @@ -14203,33 +14990,40 @@ components: schema: type: string format: date-time - CreatedOrStartedAfter: - name: created_or_started_after - description: - filter on created_at for non non started job and started_at otherwise - after (exclusive) timestamp + + CompletedBefore: + name: completed_before + description: filter on started before (inclusive) timestamp in: query schema: type: string format: date-time - CreatedOrStartedAfterCompletedJob: - name: created_or_started_after_completed_jobs - description: - filter on created_at for non non started job and started_at otherwise - after (exclusive) timestamp but only for the completed jobs + CompletedAfter: + name: completed_after + description: filter on started after (exclusive) timestamp in: query schema: type: string format: date-time - CreatedOrStartedBefore: - name: created_or_started_before + + CreatedAfterQueue: + name: created_after_queue description: - filter on created_at for non non started job and started_at otherwise - before (inclusive) timestamp + filter on jobs created after X for jobs in the queue only in: query schema: type: string format: date-time + + CreatedBeforeQueue: + name: created_before_queue + description: + filter on jobs created before X for jobs in the queue only + in: query + schema: + type: string + format: date-time + Success: name: success description: filter on successful jobs @@ -14408,6 +15202,72 @@ components: # -- INLINE END -- # Do not change line above + FlowConversation: + type: object + required: + [id, workspace_id, flow_path, created_at, updated_at, created_by] + properties: + id: + type: string + format: uuid + description: Unique identifier for the conversation + workspace_id: + type: string + description: The workspace ID where the conversation belongs + flow_path: + type: string + description: Path of the flow this conversation is for + title: + type: string + description: Optional title for the conversation + nullable: true + created_at: + type: string + format: date-time + description: When the conversation was created + updated_at: + type: string + format: date-time + description: When the conversation was last updated + created_by: + type: string + description: Username who created the conversation + + FlowConversationMessage: + type: object + required: [id, conversation_id, message_type, content, created_at] + properties: + id: + type: string + format: uuid + description: Unique identifier for the message + conversation_id: + type: string + format: uuid + description: The conversation this message belongs to + message_type: + type: string + enum: [user, assistant, system, tool] + description: Type of the message + content: + type: string + description: The message content + job_id: + type: string + format: uuid + nullable: true + description: Associated job ID if this message came from a flow run + created_at: + type: string + format: date-time + description: When the message was created + step_name: + type: string + description: The step name that produced that message + success: + type: boolean + description: Whether the message is a success + EndpointTool: type: object required: [name, description, instructions, path, method] @@ -14513,6 +15373,12 @@ components: type: object additionalProperties: type: string + max_tokens_per_model: + type: object + additionalProperties: + type: integer + minimum: 1 + maximum: 2000000 Alert: type: object @@ -14610,6 +15476,10 @@ components: type: integer concurrency_key: type: string + debounce_key: + type: string + debounce_delay_s: + type: integer cache_ttl: type: number dedicated_worker: @@ -14707,6 +15577,10 @@ components: type: string concurrency_key: type: string + debounce_key: + type: string + debounce_delay_s: + type: integer visible_to_runner_only: type: boolean no_main_func: @@ -14881,7 +15755,7 @@ components: "script_hub", "identity", "deploymentcallback", - "singlescriptflow", + "singlestepflow", "flowscript", "flownode", "appscript", @@ -14954,6 +15828,9 @@ components: started_at: type: string format: date-time + completed_at: + type: string + format: date-time duration_ms: type: integer success: @@ -14991,7 +15868,7 @@ components: "script_hub", "identity", "deploymentcallback", - "singlescriptflow", + "singlestepflow", "flowscript", "flownode", "appscript", @@ -15337,6 +16214,8 @@ components: AuditLog: type: object properties: + workspace_id: + type: string id: type: integer timestamp: @@ -15442,6 +16321,7 @@ components: span: type: string required: + - workspace_id - id - timestamp - username @@ -15584,30 +16464,30 @@ components: ScriptLang: type: string enum: [ - python3, - deno, - go, - bash, - powershell, - postgresql, - mysql, - bigquery, - snowflake, - mssql, - oracledb, - graphql, - nativets, - bun, - php, - rust, - ansible, - csharp, - nu, - java, - ruby, - duckdb, - # for related places search: ADD_NEW_LANG - ] + python3, + deno, + go, + bash, + powershell, + postgresql, + mysql, + bigquery, + snowflake, + mssql, + oracledb, + graphql, + nativets, + bun, + php, + rust, + ansible, + csharp, + nu, + java, + ruby, + duckdb, + # for related places search: ADD_NEW_LANG + ] Preview: type: object @@ -15857,6 +16737,9 @@ components: format: date-time cron_version: type: string + dynamic_skip: + type: string + description: Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean. required: - path - edited_by @@ -15975,6 +16858,9 @@ components: cron_version: type: string description: The version of the cron schedule to use (last is v2) + dynamic_skip: + type: string + description: Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean. required: - path - schedule @@ -16048,6 +16934,9 @@ components: cron_version: type: string description: The version of the cron schedule to use (last is v2) + dynamic_skip: + type: string + description: Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean. required: - schedule - timezone @@ -16204,6 +17093,13 @@ components: - delete - patch + HttpRequestType: + type: string + enum: + - sync + - async + - sync_sse + HttpTrigger: allOf: - $ref: "#/components/schemas/TriggerExtraProperty" @@ -16230,8 +17126,8 @@ components: type: string description: type: string - is_async: - type: boolean + request_type: + $ref: "#/components/schemas/HttpRequestType" authentication_method: $ref: "#/components/schemas/AuthenticationMethod" is_static_website: @@ -16251,7 +17147,7 @@ components: required: - route_path - - is_async + - request_type - authentication_method - http_method - is_static_website @@ -16293,6 +17189,9 @@ components: type: string is_async: type: boolean + description: Deprecated, use request_type instead + request_type: + $ref: "#/components/schemas/HttpRequestType" authentication_method: $ref: "#/components/schemas/AuthenticationMethod" is_static_website: @@ -16313,7 +17212,6 @@ components: - script_path - route_path - is_flow - - is_async - authentication_method - http_method - is_static_website @@ -16352,6 +17250,9 @@ components: $ref: "#/components/schemas/HttpMethod" is_async: type: boolean + description: Deprecated, use request_type instead + request_type: + $ref: "#/components/schemas/HttpRequestType" authentication_method: $ref: "#/components/schemas/AuthenticationMethod" is_static_website: @@ -16371,7 +17272,6 @@ components: - script_path - is_flow - kind - - is_async - authentication_method - http_method - is_static_website @@ -16444,6 +17344,8 @@ components: $ref: "#/components/schemas/ScriptArgs" can_return_message: type: boolean + can_return_error_result: + type: boolean error_handler_path: type: string error_handler_args: @@ -16456,6 +17358,7 @@ components: - enabled - filters - can_return_message + - can_return_error_result NewWebsocketTrigger: type: object @@ -16489,6 +17392,8 @@ components: $ref: "#/components/schemas/ScriptArgs" can_return_message: type: boolean + can_return_error_result: + type: boolean error_handler_path: type: string error_handler_args: @@ -16503,6 +17408,7 @@ components: - is_flow - filters - can_return_message + - can_return_error_result EditWebsocketTrigger: type: object @@ -16534,6 +17440,8 @@ components: $ref: "#/components/schemas/ScriptArgs" can_return_message: type: boolean + can_return_error_result: + type: boolean error_handler_path: type: string error_handler_args: @@ -16548,6 +17456,7 @@ components: - is_flow - filters - can_return_message + - can_return_error_result WebsocketTriggerInitialMessage: anyOf: - type: object @@ -16817,6 +17726,12 @@ components: type: boolean auto_acknowledge_msg: type: boolean + ack_deadline: + type: integer + format: int32 + minimum: 10 + maximum: 600 + description: "Time in seconds within which the message must be acknowledged. If not provided, defaults to the subscription's acknowledgment deadline (600 seconds)." error_handler_path: type: string error_handler_args: @@ -16890,6 +17805,48 @@ components: - enabled - aws_auth_resource_type + LoggedWizardStatus: + type: string + enum: + - OK + - SKIP + - FAIL + + DucklakeInstanceCatalogDbStatusLogs: + type: object + properties: + super_admin: + $ref: "#/components/schemas/LoggedWizardStatus" + database_credentials: + $ref: "#/components/schemas/LoggedWizardStatus" + valid_dbname: + $ref: "#/components/schemas/LoggedWizardStatus" + created_database: + $ref: "#/components/schemas/LoggedWizardStatus" + description: Created database status log + db_connect: + $ref: "#/components/schemas/LoggedWizardStatus" + grant_permissions: + $ref: "#/components/schemas/LoggedWizardStatus" + + DucklakeInstanceCatalogDbStatus: + type: object + required: + - logs + - success + properties: + logs: + $ref: "#/components/schemas/DucklakeInstanceCatalogDbStatusLogs" + success: + type: boolean + description: Whether the operation completed successfully + example: true + error: + type: string + nullable: true + description: Error message if the operation failed + example: "Connection timeout" + NewSqsTrigger: type: object properties: @@ -17584,11 +18541,14 @@ components: created_by: type: string nullable: true + disabled: + type: boolean required: - id - name - username - color + - disabled required: - email - workspaces @@ -17615,12 +18575,8 @@ components: type: string name: type: string - username: - type: string color: type: string - parent_workspace_id: - type: string required: - id - name @@ -17648,6 +18604,25 @@ components: - owner - created_at + DependencyMap: + type: object + properties: + workspace_id: + type: string + nullable: true + importer_path: + type: string + nullable: true + importer_kind: + type: string + nullable: true + imported_path: + type: string + nullable: true + importer_node_id: + type: string + nullable: true + WorkspaceInvite: type: object properties: @@ -17690,12 +18665,15 @@ components: type: string operator_only: type: boolean + first_time_user: + type: boolean required: - email - login_type - super_admin - verified + - first_time_user Flow: allOf: @@ -18087,6 +19065,10 @@ components: type: string public_resource: type: boolean + advanced_permissions: + type: array + items: + $ref: "#/components/schemas/S3PermissionRule" secondary_storage: type: object additionalProperties: @@ -18144,6 +19126,49 @@ components: required: - path + DynamicInputData: + type: object + properties: + entrypoint_function: + type: string + description: Name of the function to execute for dynamic select + args: + type: object + description: Arguments to pass to the function + runnable_ref: + type: object + oneOf: + - type: object + properties: + source: + type: string + enum: [deployed] + path: + type: string + description: Path to the deployed script or flow + runnable_kind: + $ref: "#/components/schemas/RunnableKind" + required: + - source + - path + - runnable_kind + - type: object + properties: + source: + type: string + enum: [inline] + code: + type: string + description: Code content for inline execution + language: + $ref: "#/components/schemas/ScriptLang" + required: + - source + - code + required: + - entrypoint_function + - runnable_ref + WindmillLargeFile: type: object properties: @@ -18240,6 +19265,15 @@ components: additionalProperties: type: string + S3PermissionRule: + type: object + properties: + pattern: + type: string + allow: + type: string # comma separated permissions : "read,write,delete,list" + required: ["pattern", "allow"] + GitRepositorySettings: type: object properties: @@ -18280,17 +19314,6 @@ components: - script_path - git_repo_resource_path - UploadFilePart: - type: object - properties: - part_number: - type: integer - tag: - type: string - required: - - part_number - - tag - MetricMetadata: type: object properties: @@ -18775,10 +19798,18 @@ components: required: - name - url + total_count: + type: number + description: Total number of repositories available for this installation + per_page: + type: number + description: Number of repositories loaded per page required: - installation_id - account_id - repositories + - total_count + - per_page WorkspaceGithubInstallation: type: object diff --git a/backend/windmill-api/src/agent_workers_oss.rs b/backend/windmill-api/src/agent_workers_oss.rs index dabb2cb079..1459efef1d 100644 --- a/backend/windmill-api/src/agent_workers_oss.rs +++ b/backend/windmill-api/src/agent_workers_oss.rs @@ -17,7 +17,7 @@ use crate::db::DB; use axum::Router; #[cfg(not(feature = "private"))] -pub fn global_service() -> Router { +pub fn global_service(_job_completed_tx: windmill_worker::JobCompletedSender) -> Router { Router::new() } diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 9f13d32cf5..e4d07cfdc6 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -1,7 +1,4 @@ -use crate::{ - db::{ApiAuthed, DB}, - variables::get_variable_or_self, -}; +use crate::db::{ApiAuthed, DB}; use axum::{body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router}; use http::{HeaderMap, Method}; @@ -9,24 +6,50 @@ use quick_cache::sync::Cache; use reqwest::{Client, RequestBuilder}; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; +use windmill_common::variables::get_variable_or_self; use std::collections::HashMap; -use windmill_common::ai_providers::{AIProvider, ProviderConfig, ProviderModel}; use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::ai_providers::{AIProvider, ProviderConfig, ProviderModel, AZURE_API_VERSION}; use windmill_common::error::{to_anyhow, Error, Result}; +use windmill_common::utils::configure_client; lazy_static::lazy_static! { - static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new() + static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() .timeout(std::time::Duration::from_secs(60 * 5)) - .user_agent("windmill/beta") + .user_agent("windmill/beta")) .build().unwrap(); static ref OPENAI_AZURE_BASE_PATH: Option = std::env::var("OPENAI_AZURE_BASE_PATH").ok(); pub static ref AI_REQUEST_CACHE: Cache<(String, AIProvider), ExpiringAIRequestConfig> = Cache::new(500); -} -const AZURE_API_VERSION: &str = "2025-04-01-preview"; -const OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; + /// Parse AI_HTTP_HEADERS environment variable into a vector of (header_name, header_value) tuples + /// Format: "header1: value1, header2: value2" + static ref AI_HTTP_HEADERS: Vec<(String, String)> = { + std::env::var("AI_HTTP_HEADERS") + .ok() + .map(|headers_str| { + headers_str + .split(',') + .filter_map(|header| { + let parts: Vec<&str> = header.splitn(2, ':').collect(); + if parts.len() == 2 { + let name = parts[0].trim().to_string(); + let value = parts[1].trim().to_string(); + if !name.is_empty() && !value.is_empty() { + Some((name, value)) + } else { + None + } + } else { + None + } + }) + .collect() + }) + .unwrap_or_default() + }; +} #[derive(Deserialize, Debug)] struct AIOAuthResource { @@ -154,21 +177,13 @@ impl AIRequestConfig { let base_url = self.base_url.trim_end_matches('/'); - let is_azure = matches!(provider, AIProvider::OpenAI) && base_url != OPENAI_BASE_URL - || matches!(provider, AIProvider::AzureOpenAI); + let is_azure = provider.is_azure_openai(base_url); let is_anthropic = matches!(provider, AIProvider::Anthropic); let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some(); let url = if is_azure && method != Method::GET { - if base_url.ends_with("/deployments") { - let model = Self::get_azure_model(&body)?; - format!("{}/{}/{}", base_url, model, path) - } else if base_url.ends_with("/openai") { - let model = Self::get_azure_model(&body)?; - format!("{}/deployments/{}/{}", base_url, model, path) - } else { - format!("{}/{}", base_url, path) - } + let model = AIProvider::extract_model_from_body(&body)?; + AIProvider::build_azure_openai_url(base_url, &model, path) } else if is_anthropic_sdk { let truncated_base_url = base_url.trim_end_matches("/v1"); format!("{}/{}", truncated_base_url, path) @@ -213,6 +228,11 @@ impl AIRequestConfig { request = request.header("OpenAI-Organization", org_id); } + // Apply custom headers from AI_HTTP_HEADERS environment variable + for (header_name, header_value) in AI_HTTP_HEADERS.iter() { + request = request.header(header_name.as_str(), header_value.as_str()); + } + Ok(request) } @@ -233,18 +253,6 @@ impl AIRequestConfig { .map_err(|e| Error::internal_err(format!("Failed to reserialize request body: {}", e)))? .into()) } - - fn get_azure_model(body: &Bytes) -> Result { - #[derive(Deserialize, Debug)] - struct AzureModel { - model: String, - } - - let azure_model: AzureModel = serde_json::from_slice(body) - .map_err(|e| Error::internal_err(format!("Failed to parse request body: {}", e)))?; - - Ok(azure_model.model) - } } #[derive(Clone, Debug)] @@ -272,6 +280,8 @@ pub struct AIConfig { pub code_completion_model: Option, #[serde(skip_serializing_if = "Option::is_none")] pub custom_prompts: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_tokens_per_model: Option>, } pub fn global_service() -> Router { @@ -310,11 +320,17 @@ async fn global_proxy( let url = format!("{}/{}", base_url, ai_path); - let request = HTTP_CLIENT + let mut request = HTTP_CLIENT .request(method, url) .header("content-type", "application/json") - .header("Authorization", format!("Bearer {}", api_key)) - .body(body); + .header("Authorization", format!("Bearer {}", api_key)); + + // Apply custom headers from AI_HTTP_HEADERS environment variable + for (header_name, header_value) in AI_HTTP_HEADERS.iter() { + request = request.header(header_name.as_str(), header_value.as_str()); + } + + let request = request.body(body); let response = request.send().await.map_err(to_anyhow)?; diff --git a/backend/windmill-api/src/approvals.rs b/backend/windmill-api/src/approvals.rs index e03b587f10..1296000481 100644 --- a/backend/windmill-api/src/approvals.rs +++ b/backend/windmill-api/src/approvals.rs @@ -204,31 +204,31 @@ pub async fn get_approval_form_details( "WITH job_info AS ( -- Query for Teams (running jobs) SELECT - parent.job_kind AS \"job_kind!: JobKind\", - parent.script_hash AS \"script_hash: ScriptHash\", - parent.raw_flow AS \"raw_flow: sqlx::types::Json>\", - child.parent_job AS \"parent_job: Uuid\", - parent.created_at AS \"created_at!: chrono::NaiveDateTime\", - parent.created_by AS \"created_by!\", - parent.script_path, - parent.args AS \"args: sqlx::types::Json>\" - FROM v2_as_queue child - JOIN v2_as_queue parent ON parent.id = child.parent_job - WHERE child.id = $1 AND child.workspace_id = $2 + parent_j.kind AS \"job_kind!: JobKind\", + parent_j.runnable_id AS \"script_hash: ScriptHash\", + parent_j.raw_flow AS \"raw_flow: sqlx::types::Json>\", + child_j.parent_job AS \"parent_job: Uuid\", + parent_j.created_at AS \"created_at!: chrono::NaiveDateTime\", + parent_j.created_by AS \"created_by!\", + parent_j.runnable_path as script_path, + parent_j.args AS \"args: sqlx::types::Json>\" + FROM v2_job_queue child_q JOIN v2_job child_j USING (id) + JOIN v2_job parent_j ON parent_j.id = child_j.parent_job + WHERE child_j.id = $1 AND child_j.workspace_id = $2 UNION ALL -- Query for Slack (completed jobs) SELECT - v2_as_queue.job_kind AS \"job_kind!: JobKind\", - v2_as_queue.script_hash AS \"script_hash: ScriptHash\", - v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json>\", - v2_as_completed_job.parent_job AS \"parent_job: Uuid\", - v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\", - v2_as_completed_job.created_by AS \"created_by!\", - v2_as_queue.script_path, - v2_as_queue.args AS \"args: sqlx::types::Json>\" - FROM v2_as_queue - JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id - WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 + parent_j.kind AS \"job_kind!: JobKind\", + parent_j.runnable_id AS \"script_hash: ScriptHash\", + parent_j.raw_flow AS \"raw_flow: sqlx::types::Json>\", + completed_j.parent_job AS \"parent_job: Uuid\", + completed_j.created_at AS \"created_at!: chrono::NaiveDateTime\", + completed_j.created_by AS \"created_by!\", + parent_j.runnable_path as script_path, + parent_j.args AS \"args: sqlx::types::Json>\" + FROM v2_job_completed completed_c JOIN v2_job completed_j USING (id) + JOIN v2_job parent_j ON parent_j.id = completed_j.parent_job + WHERE completed_j.id = $1 AND completed_j.workspace_id = $2 ) SELECT * FROM job_info LIMIT 1", job_id, diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 2aef45acb5..b669f5e9a9 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -7,7 +7,6 @@ use std::{collections::HashMap, sync::Arc}; * Please see the included NOTICE for copyright information and * LICENSE-AGPL for a copy of the license. */ - use crate::{ auth::OptTokened, db::{ApiAuthed, DB}, @@ -21,7 +20,7 @@ use crate::{ use crate::{ job_helpers_oss::{ download_s3_file_internal, get_random_file_name, get_s3_resource, - get_workspace_s3_resource, upload_file_from_req, DownloadFileQuery, + get_workspace_s3_resource_and_check_paths, upload_file_from_req, DownloadFileQuery, }, users::fetch_api_authed_from_permissioned_as, }; @@ -61,7 +60,7 @@ use windmill_common::{ users::username_to_permissioned_as, utils::{ http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, - Pagination, RunnableKind, StripPath, + Pagination, RunnableKind, StripPath, WarnAfterExt, }, variables::{build_crypt, build_crypt_with_key_suffix, encrypt}, worker::{to_raw_value, CLOUD_HOSTED}, @@ -77,7 +76,7 @@ use hmac::Mac; use windmill_common::{ jwt, oauth2::HmacSha256, - s3_helpers::{build_object_store_client, S3Object}, + s3_helpers::{build_object_store_client, S3Object, S3Permission}, variables::get_workspace_key, }; @@ -89,6 +88,10 @@ pub fn workspaced_service() -> Router { .route("/get/lite/*path", get(get_app_lite)) .route("/get/draft/*path", get(get_app_w_draft)) .route("/secret_of/*path", get(get_secret_id)) + .route( + "/secret_of_latest_version/*path", + get(get_latest_version_secret_id), + ) .route("/get/v/*id", get(get_app_by_id)) .route("/get_data/v/*id", get(get_raw_app_data)) .route("/exists/*path", get(exists_app)) @@ -148,14 +151,14 @@ fn is_false(b: &bool) -> bool { !b } -#[derive(FromRow, Serialize, Deserialize)] -pub struct AppVersion { - pub id: i64, - pub app_id: Uuid, - pub value: sqlx::types::Json>, - pub created_by: String, - pub created_at: chrono::DateTime, -} +// #[derive(FromRow, Serialize, Deserialize)] +// pub struct AppVersion { +// pub id: i64, +// pub app_id: Uuid, +// pub value: sqlx::types::Json>, +// pub created_by: String, +// pub created_at: chrono::DateTime, +// } #[derive(Debug, Serialize, Deserialize, FromRow)] pub struct AppWithLastVersion { @@ -181,15 +184,6 @@ pub struct AppWithLastVersionAndStarred { pub starred: Option, } -#[cfg(feature = "enterprise")] -#[derive(Debug, Serialize, FromRow)] -pub struct AppWithLastVersionAndWorkspace { - #[sqlx(flatten)] - #[serde(flatten)] - pub app: AppWithLastVersion, - pub workspace_id: String, -} - #[derive(Serialize, Deserialize, FromRow)] pub struct AppWithLastVersionAndDraft { #[sqlx(flatten)] @@ -278,7 +272,7 @@ pub struct CreateApp { pub custom_path: Option, } -#[derive(Deserialize)] +#[derive(Serialize, Deserialize)] pub struct EditApp { pub path: Option, pub summary: Option, @@ -399,19 +393,83 @@ async fn list_apps( Ok(Json(rows)) } -async fn get_raw_app_data(Path((w_id, version_id)): Path<(String, String)>) -> Result { - let file_path = format!("/tmp/wmill/{}/{}", w_id, version_id); - let file = tokio::fs::File::open(file_path).await?; - let stream = tokio_util::io::ReaderStream::new(file); - let res = Response::builder().header( - http::header::CONTENT_TYPE, - if version_id.ends_with(".css") { - "text/css" - } else { - "text/javascript" - }, - ); - Ok(res.body(Body::from_stream(stream)).unwrap()) +async fn get_raw_app_data( + Path((w_id, secret_with_ext)): Path<(String, String)>, + Extension(db): Extension, +) -> Result { + #[cfg(all(feature = "enterprise", feature = "parquet"))] + let object_store = windmill_common::s3_helpers::get_object_store().await; + + // tracing::info!("secret_with_ext: {}", secret_with_ext); + let mut splitted = secret_with_ext.split('.'); + let secret_id = splitted.next().unwrap_or(""); + + if secret_id.is_empty() { + return Err(Error::BadRequest("Invalid secret".to_string())); + } + + let id = get_id_from_secret( + &db, + &w_id, + secret_id.to_string(), + Some(BUNDLE_SECRET_PREFIX), + ) + .await?; + + let file_type = splitted.next().unwrap_or(""); + let file_type = if file_type == "css" { + "css" + } else if file_type == "js" { + "js" + } else { + return Err(Error::BadRequest( + "Invalid file type, only .css and .js are supported".to_string(), + )); + }; + // tracing::info!("file_type: {}", file_type); + + #[allow(unused_assignments)] + let mut body: Option = None; + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if let Some(os) = object_store { + let path = format!("/app_bundles/{}/{}.{}", w_id, id, file_type); + let stream = os + .get(&object_store::path::Path::from(path)) + .await? + .bytes() + .await?; + tracing::info!("stream: {}", stream.len()); + body = Some(Body::from(stream)); + } + + if body.is_none() { + let get_raw_app_file = sqlx::query_scalar!( + "SELECT data FROM app_bundles WHERE app_version_id = $1 AND file_type = $2 AND w_id = $3", + id, + file_type, + &w_id, + ) + .fetch_optional(&db) + .await?; + if let Some(file) = get_raw_app_file { + body = Some(Body::from(file)); + } + } + + if let Some(body) = body { + // let stream = tokio_util::io::ReaderStream::new(file); + let res = Response::builder().header( + http::header::CONTENT_TYPE, + if file_type == "css" { + "text/css" + } else { + "text/javascript" + }, + ); + Ok(res.body(body).unwrap()) + } else { + return Err(Error::NotFound("File not found".to_string())); + } } // async fn get_app_version( @@ -641,7 +699,7 @@ async fn update_app_history( check_scopes(&authed, || format!("apps:write:{}", &app_path))?; sqlx::query!( - "INSERT INTO deployment_metadata (workspace_id, path, app_version, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path, app_version) WHERE app_version IS NOT NULL DO UPDATE SET deployment_msg = $4", + "INSERT INTO deployment_metadata (workspace_id, path, app_version, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path, app_version) WHERE app_version IS NOT NULL DO UPDATE SET deployment_msg = EXCLUDED.deployment_msg", w_id, app_path, app_version, @@ -702,14 +760,7 @@ async fn get_public_app_by_secret( Extension(db): Extension, Path((w_id, secret)): Path<(String, String)>, ) -> JsonResult { - let mc = build_crypt(&db, &w_id).await?; - - let decrypted = mc - .decrypt_bytes_to_bytes(&(hex::decode(secret)?)) - .map_err(|e| Error::internal_err(e.to_string()))?; - let bytes = str::from_utf8(&decrypted).map_err(to_anyhow)?; - - let id: i64 = bytes.parse().map_err(to_anyhow)?; + let id = get_id_from_secret(&db, &w_id, secret, None).await?; let app_o = sqlx::query_as::<_, AppWithLastVersion>( "SELECT app.id, app.path, app.summary, app.versions, app.policy, app.custom_path, @@ -757,6 +808,27 @@ async fn get_public_app_by_secret( Ok(Json(app)) } +async fn get_id_from_secret( + db: &DB, + w_id: &str, + secret: String, + prefix: Option<&str>, +) -> Result { + let mc = build_crypt(db, w_id).await?; + let decrypted = mc + .decrypt_bytes_to_bytes(&(hex::decode(secret)?)) + .map_err(|e| Error::internal_err(e.to_string()))?; + let mut bytes = str::from_utf8(&decrypted).map_err(to_anyhow)?; + if let Some(prefix) = prefix { + if !bytes.starts_with(prefix) { + return Err(Error::BadRequest("Invalid secret".to_string())); + } + bytes = bytes.strip_prefix(prefix).unwrap_or(""); + } + let id: i64 = bytes.parse().map_err(to_anyhow)?; + Ok(id) +} + async fn get_public_resource( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, @@ -813,16 +885,85 @@ async fn get_secret_id( Ok(hx) } +const BUNDLE_SECRET_PREFIX: &str = "bundle_"; + +async fn get_latest_version_secret_id( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || format!("apps:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + + let id_o = sqlx::query_scalar!( + "SELECT app.versions[array_upper(app.versions, 1)] FROM app + WHERE app.path = $1 AND app.workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten(); + + tx.commit().await?; + + let id = not_found_if_none(id_o, "App", path.to_string())?; + + let mc = build_crypt(&db, &w_id).await?; + + let hx = hex::encode(mc.encrypt_str_to_bytes(format!("{}{}", BUNDLE_SECRET_PREFIX, id))); + + Ok(hx) +} + +async fn store_raw_app_file<'a>( + w_id: &str, + id: &i64, + file_type: &str, + data: bytes::Bytes, + tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, +) -> Result<()> { + #[cfg(all(feature = "enterprise", feature = "parquet"))] + { + let object_store = windmill_common::s3_helpers::get_object_store().await; + + let path: String = format!("/app_bundles/{}/{}.{}", w_id, id, file_type); + + if let Some(os) = object_store { + if let Err(e) = os + .put(&object_store::path::Path::from(path.clone()), data.into()) + .await + { + tracing::error!("Failed to put snapshot to s3 at {path}: {:?}", e); + return Err(windmill_common::error::Error::ExecutionErr(format!( + "Failed to put {path} to s3" + ))); + } + tracing::info!("Successfully put snapshot to s3 at {path}"); + return Ok(()); + } + } + + sqlx::query!( + "INSERT INTO app_bundles (app_version_id, w_id, file_type, data) VALUES ($1, $2, $3, $4)", + id, + w_id, + file_type, + data.to_vec() + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} macro_rules! process_app_multipart { ($authed:expr, $user_db:expr, $db:expr, $w_id:expr, $path:expr, $multipart:expr, $internal_fn:expr) => { async { let mut saved_app = None; let mut uploaded_js = false; - //todo: use s3 instead - let file_path = format!("/tmp/wmill/{}", $w_id); - std::fs::create_dir_all(&file_path).unwrap(); - let mut multipart = $multipart; while let Some(field) = multipart.next_field().await.unwrap() { let name = field.name().unwrap().to_string(); @@ -841,9 +982,8 @@ macro_rules! process_app_multipart { .await?; saved_app = Some((npath, nid, ntx)); } else if name == "js" { - if let Some((_npath, id, _tx)) = saved_app.as_ref() { - let file_path = format!("{}/{}.js", file_path, id); - std::fs::write(file_path, data).unwrap(); + if let Some((_npath, id, tx)) = saved_app.as_mut() { + store_raw_app_file($w_id, &id, "js", data, tx).await?; uploaded_js = true; } else { return Err(Error::BadRequest( @@ -851,9 +991,8 @@ macro_rules! process_app_multipart { )); } } else if name == "css" { - if let Some((_npath, id, _tx)) = saved_app.as_ref() { - let file_path = format!("{}/{}.css", file_path, id); - std::fs::write(file_path, data).unwrap(); + if let Some((_npath, id, tx)) = saved_app.as_mut() { + store_raw_app_file($w_id, &id, "css", data, tx).await?; } else { return Err(Error::BadRequest( "App payload need to be created first".to_string(), @@ -1100,6 +1239,8 @@ async fn create_app_internal<'a>( None, Some(&authed.clone().into()), false, + None, + None, ) .await?; tracing::info!("Pushed app dependency job {}", dependency_job_uuid); @@ -1383,6 +1524,14 @@ async fn update_app_internal<'a>( path.to_owned() }; let v_id = if let Some(nvalue) = &ns.value { + // Row lock debounce key for path. We need this to make all updates of runnables sequential and predictable. + tokio::time::timeout( + core::time::Duration::from_secs(60), + windmill_common::jobs::lock_debounce_key(&w_id, &npath, &mut tx), + ) + .warn_after_seconds(10) + .await??; + let app_id = sqlx::query_scalar!( "SELECT id FROM app WHERE path = $1 AND workspace_id = $2", npath, @@ -1479,6 +1628,8 @@ async fn update_app_internal<'a>( None, Some(&authed.clone().into()), false, + None, + None, ) .await?; tracing::info!("Pushed app dependency job {}", dependency_job_uuid); @@ -1765,6 +1916,8 @@ async fn execute_component( (email.as_str(), permissioned_as) }; + let end_user_email = opt_authed.as_ref().map(|a| a.email.clone()); + let (uuid, tx) = push( &db, tx, @@ -1794,6 +1947,8 @@ async fn execute_component( None, None, false, + end_user_email, + None, ) .await?; tx.commit().await?; @@ -1973,61 +2128,86 @@ async fn upload_s3_file_from_app( let user_db = UserDB::new(db.clone()); - let (s3_resource_opt, file_key, on_behalf_of_email, permissioned_as, username) = if policy - .as_ref() - .is_some_and(|p| p.s3_inputs.is_some()) - { - let policy = policy.unwrap(); - let s3_inputs = policy.s3_inputs.as_ref().unwrap(); + let (s3_resource_opt, file_key, on_behalf_of_email, permissioned_as, username) = + if policy.as_ref().is_some_and(|p| p.s3_inputs.is_some()) { + let policy = policy.unwrap(); + let s3_inputs = policy.s3_inputs.as_ref().unwrap(); - let (username, permissioned_as, email) = - get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; + let (username, permissioned_as, email) = + get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; - let on_behalf_authed = fetch_api_authed_from_permissioned_as( - permissioned_as.clone(), - email.clone(), - &w_id, - &db, - Some(username.clone()), - ) - .await?; + let on_behalf_authed = fetch_api_authed_from_permissioned_as( + permissioned_as.clone(), + email.clone(), + &w_id, + &db, + Some(username.clone()), + ) + .await?; - if let Some(file_key) = query.file_key { - // file key is provided => requires workspace, user or list policy and must match the regex - let matching_s3_inputs = if let Some(ref s3_resource_path) = query.s3_resource_path { - s3_inputs - .iter() - .filter(|s3_input| { - s3_input.allowed_resources.contains(s3_resource_path) - || s3_input.allow_user_resources - }) - .sorted_by_key(|i| i.allow_user_resources) // consider user resources last - .collect::>() - } else { - s3_inputs - .iter() - .filter(|s3_input| s3_input.allow_workspace_resource) - .collect::>() - }; + if let Some(file_key) = query.file_key { + // file key is provided => requires workspace, user or list policy and must match the regex + let matching_s3_inputs = if let Some(ref s3_resource_path) = query.s3_resource_path + { + s3_inputs + .iter() + .filter(|s3_input| { + s3_input.allowed_resources.contains(s3_resource_path) + || s3_input.allow_user_resources + }) + .sorted_by_key(|i| i.allow_user_resources) // consider user resources last + .collect::>() + } else { + s3_inputs + .iter() + .filter(|s3_input| s3_input.allow_workspace_resource) + .collect::>() + }; - let matched_input = matching_s3_inputs.iter().find(|s3_input| { - match Regex::new(&s3_input.file_key_regex) { - Ok(re) => re.is_match(&file_key), - Err(e) => { - tracing::error!("Error compiling regex: {}", e); - false + let matched_input = matching_s3_inputs.iter().find(|s3_input| { + match Regex::new(&s3_input.file_key_regex) { + Ok(re) => re.is_match(&file_key), + Err(e) => { + tracing::error!("Error compiling regex: {}", e); + false + } } - } - }); + }); - if let Some(matched_input) = matched_input { - if let Some(ref s3_resource_path) = query.s3_resource_path { - if matched_input.allow_user_resources { - if let Some(authed) = opt_authed { + if let Some(matched_input) = matched_input { + if let Some(ref s3_resource_path) = query.s3_resource_path { + if matched_input.allow_user_resources { + if let Some(authed) = opt_authed { + ( + Some( + get_s3_resource( + &authed, + &db, + Some(user_db), + "", + &w_id, + s3_resource_path, + None, + None, + ) + .await?, + ), + file_key, + email, + permissioned_as, + username, + ) + } else { + return Err(Error::BadRequest( + "User resources are not allowed without being logged in" + .to_string(), + )); + } + } else { ( Some( get_s3_resource( - &authed, + &on_behalf_authed, &db, Some(user_db), "", @@ -2043,115 +2223,112 @@ async fn upload_s3_file_from_app( permissioned_as, username, ) - } else { - return Err(Error::BadRequest( - "User resources are not allowed without being logged in" - .to_string(), - )); } } else { - ( - Some( - get_s3_resource( - &on_behalf_authed, - &db, - Some(user_db), - "", - &w_id, - s3_resource_path, - None, - None, - ) - .await?, - ), - file_key, - email, - permissioned_as, - username, - ) - } - } else { - let (_, s3_resource_opt) = - get_workspace_s3_resource(&on_behalf_authed, &db, None, "", &w_id, None) - .await?; - (s3_resource_opt, file_key, email, permissioned_as, username) - } - } else { - return Err(Error::BadRequest( - "No matching s3 resource found for the given file key".to_string(), - )); - } - } else { - // no file key => requires unnamed upload policy => allow workspace resource and file_key_regex is empty - let has_unnamed_policy = s3_inputs.iter().any(|s3_input| { - s3_input.allow_workspace_resource && s3_input.file_key_regex.is_empty() - }); - - if !has_unnamed_policy { - return Err(Error::BadRequest( - "no policy found for unnamed s3 file upload".to_string(), - )); - } - - // for now, we place all files into `windmill_uploads` folder with a random name - // TODO: make the folder configurable via the workspace settings - let file_key = get_random_file_name(query.file_extension); - - let (_, s3_resource_opt) = - get_workspace_s3_resource(&on_behalf_authed, &db, None, "", &w_id, None).await?; - - (s3_resource_opt, file_key, email, permissioned_as, username) - } - } else { - // backward compatibility (no policy) - // if no policy but logged in, use the user's auth to get the s3 resource - if let Some(authed) = opt_authed { - let file_key = query - .file_key - .unwrap_or_else(|| get_random_file_name(query.file_extension)); - - let (on_behalf_of_email, permissioned_as, username) = ( - authed.email.clone(), - username_to_permissioned_as(&authed.username), - authed.display_username().to_string(), - ); - - if let Some(ref s3_resource_path) = query.s3_resource_path { - ( - Some( - get_s3_resource( - &authed, + let (_, s3_resource_opt) = get_workspace_s3_resource_and_check_paths( + &on_behalf_authed, &db, - Some(user_db), + None, "", &w_id, - s3_resource_path, - None, None, + &[(&file_key, S3Permission::WRITE)], ) - .await?, - ), - file_key, - on_behalf_of_email, - permissioned_as, - username, - ) + .await?; + (s3_resource_opt, file_key, email, permissioned_as, username) + } + } else { + return Err(Error::BadRequest( + "No matching s3 resource found for the given file key".to_string(), + )); + } } else { - let (_, s3_resource) = - get_workspace_s3_resource(&authed, &db, None, "", &w_id, None).await?; + // no file key => requires unnamed upload policy => allow workspace resource and file_key_regex is empty + let has_unnamed_policy = s3_inputs.iter().any(|s3_input| { + s3_input.allow_workspace_resource && s3_input.file_key_regex.is_empty() + }); - ( - s3_resource, - file_key, - on_behalf_of_email, - permissioned_as, - username, + if !has_unnamed_policy { + return Err(Error::BadRequest( + "no policy found for unnamed s3 file upload".to_string(), + )); + } + + // for now, we place all files into `windmill_uploads` folder with a random name + // TODO: make the folder configurable via the workspace settings + let file_key = get_random_file_name(query.file_extension); + + let (_, s3_resource_opt) = get_workspace_s3_resource_and_check_paths( + &on_behalf_authed, + &db, + None, + "", + &w_id, + None, + &[(&file_key, S3Permission::WRITE)], ) + .await?; + + (s3_resource_opt, file_key, email, permissioned_as, username) } } else { - return Err(Error::BadRequest("Missing s3 policy".to_string())); - } - }; + // backward compatibility (no policy) + // if no policy but logged in, use the user's auth to get the s3 resource + if let Some(authed) = opt_authed { + let file_key = query + .file_key + .unwrap_or_else(|| get_random_file_name(query.file_extension)); + + let (on_behalf_of_email, permissioned_as, username) = ( + authed.email.clone(), + username_to_permissioned_as(&authed.username), + authed.display_username().to_string(), + ); + + if let Some(ref s3_resource_path) = query.s3_resource_path { + ( + Some( + get_s3_resource( + &authed, + &db, + Some(user_db), + "", + &w_id, + s3_resource_path, + None, + None, + ) + .await?, + ), + file_key, + on_behalf_of_email, + permissioned_as, + username, + ) + } else { + let (_, s3_resource) = get_workspace_s3_resource_and_check_paths( + &authed, + &db, + None, + "", + &w_id, + None, + &[(&file_key, S3Permission::WRITE)], + ) + .await?; + + ( + s3_resource, + file_key, + on_behalf_of_email, + permissioned_as, + username, + ) + } + } else { + return Err(Error::BadRequest("Missing s3 policy".to_string())); + } + }; let s3_resource = s3_resource_opt.ok_or(Error::internal_err( "No files storage resource defined at the workspace level".to_string(), @@ -2213,6 +2390,9 @@ async fn delete_s3_file_from_app( .. } = jwt::decode_with_internal_secret::(&query.delete_token).await?; + let path = object_store::path::Path::parse(file_key.as_str()) + .map_err(|e| Error::internal_err(format!("Error parsing file key: {}", e)))?; + if workspace != w_id { return Err(Error::BadRequest("Invalid workspace".to_string())); } @@ -2239,8 +2419,16 @@ async fn delete_s3_file_from_app( ) .await? } else { - let (_, s3_resource) = - get_workspace_s3_resource(&on_behalf_authed, &db, None, "", &w_id, None).await?; + let (_, s3_resource) = get_workspace_s3_resource_and_check_paths( + &on_behalf_authed, + &db, + None, + "", + &w_id, + None, + &[(&path.to_string(), S3Permission::DELETE)], + ) + .await?; s3_resource.ok_or(Error::internal_err( "No files storage resource defined at the workspace level".to_string(), @@ -2249,9 +2437,6 @@ async fn delete_s3_file_from_app( let s3_client = build_object_store_client(&s3_resource).await?; - let path = object_store::path::Path::parse(file_key.as_str()) - .map_err(|e| Error::internal_err(format!("Error parsing file key: {}", e)))?; - s3_client.delete(&path).await.map_err(|err| { tracing::error!("Error deleting file: {:?}", err); Error::internal_err(format!("Error deleting file: {}", err.to_string())) @@ -2345,13 +2530,13 @@ async fn check_if_allowed_to_access_s3_file_from_app( || { sqlx::query_scalar!( r#"SELECT EXISTS ( - SELECT 1 FROM v2_as_completed_job - WHERE workspace_id = $2 - AND (job_kind = 'appscript' OR job_kind = 'preview') - AND created_by = 'anonymous' - AND started_at > now() - interval '3 hours' - AND script_path LIKE $3 || '/%' - AND result @> ('{"s3":"' || $1 || '"}')::jsonb + SELECT 1 FROM v2_job_completed c JOIN v2_job j USING (id) + WHERE j.workspace_id = $2 + AND (j.kind = 'appscript' OR j.kind = 'preview') + AND j.created_by = 'anonymous' + AND c.started_at > now() - interval '3 hours' + AND j.runnable_path LIKE $3 || '/%' + AND c.result @> ('{"s3":"' || $1 || '"}')::jsonb )"#, file_query.s3, w_id, @@ -2517,7 +2702,7 @@ async fn build_args( &path, None, "", - false + false, ) .await?; if res.is_none() { diff --git a/backend/windmill-api/src/args.rs b/backend/windmill-api/src/args.rs index a541f8c738..763074948f 100644 --- a/backend/windmill-api/src/args.rs +++ b/backend/windmill-api/src/args.rs @@ -1,12 +1,12 @@ use std::collections::HashMap; use axum::{ - extract::{FromRequest, FromRequestParts, Multipart, Query, Request}, - http::{HeaderMap, Uri}, + extract::{FromRequest, Multipart, Query, Request}, + http::HeaderMap, response::{IntoResponse, Response}, }; use bytes::Bytes; -use http::{header::CONTENT_TYPE, request::Parts, StatusCode}; +use http::{header::CONTENT_TYPE, StatusCode}; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use sqlx::types::JsonRawValue; @@ -44,11 +44,13 @@ pub enum Body { #[derive(Debug, Clone, Default)] pub struct WebhookArgsMetadata { pub raw_string: Option, - pub headers: HashMap>, + pub headers: HeaderMap, + pub query: Option, pub method: http::Method, - pub query: HashMap>, pub query_wrap_body: bool, pub query_use_raw: bool, + pub query_include_header: Option, + pub query_include_query: Option, } pub struct RawWebhookArgs { @@ -262,6 +264,18 @@ impl WebhookArgs { self, runnable_format: RunnableFormat, ) -> Result { + let headers = build_headers( + &self.metadata.headers, + self.metadata.query_include_header, + runnable_format.has_preprocessor, + ); + + let query = build_query( + self.metadata.query.as_deref(), + self.metadata.query_include_query, + runnable_format.has_preprocessor, + ); + match runnable_format { RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V2 } => { let mut args = HashMap::new(); @@ -272,8 +286,8 @@ impl WebhookArgs { kind: "webhook".to_string(), body: to_raw_value(&self.body), raw_string: self.metadata.raw_string, - headers: self.metadata.headers, - query: self.metadata.query, + headers, + query, }), ); @@ -282,8 +296,7 @@ impl WebhookArgs { RunnableFormat { has_preprocessor, .. } => { let mut extra = HashMap::new(); - let WebhookArgsMetadata { query, query_wrap_body, headers, raw_string, .. } = - self.metadata; + let WebhookArgsMetadata { query_wrap_body, raw_string, .. } = self.metadata; for (k, v) in headers { extra.insert(k, v); @@ -332,6 +345,7 @@ pub struct RequestQuery { pub raw: Option, pub wrap_body: Option, pub include_header: Option, + pub include_query: Option, } async fn req_to_string( @@ -359,23 +373,21 @@ where let content_type = content_type_header.and_then(|value| value.to_str().ok()); let uri = request.uri(); let request_query = Query::::try_from_uri(uri).unwrap().0; - let headers = build_headers(&headers_map, request_query.include_header, is_http_trigger); - let query_decode = DecodeQueries::from_uri(uri, is_http_trigger); - let mut query = HashMap::new(); - if let Some(DecodeQueries(queries)) = query_decode { - query.extend(queries); - } + + let query = uri.query().map(|s| s.to_owned()); let raw = !is_http_trigger && request_query.raw.unwrap_or(false); let wrap_body = !is_http_trigger && request_query.wrap_body.unwrap_or(false); ( content_type, WebhookArgsMetadata { - headers, + headers: headers_map.clone(), query, method: request.method().clone(), raw_string: None, query_wrap_body: wrap_body, query_use_raw: raw, + query_include_header: request_query.include_header, + query_include_query: request_query.include_query, }, ) }; @@ -460,11 +472,11 @@ lazy_static::lazy_static! { pub fn build_headers( headers: &HeaderMap, include_header: Option, - is_http_trigger: bool, + include_all_headers: bool, ) -> HashMap> { let mut selected_headers = HashMap::new(); - if is_http_trigger { + if include_all_headers { for (k, v) in headers.iter() { selected_headers.insert( k.to_string(), @@ -490,73 +502,40 @@ pub fn build_headers( selected_headers } -#[derive(Deserialize)] -pub struct IncludeQuery { - pub include_query: Option, -} +pub fn build_query( + query: Option<&str>, + include_query: Option, + include_all_query: bool, +) -> HashMap> { + let Some(query) = query else { + return HashMap::new(); + }; -pub struct DecodeQueries(pub HashMap>); - -#[axum::async_trait] -impl FromRequestParts for DecodeQueries -where - S: Send + Sync, -{ - type Rejection = Response; - - async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { - Ok(DecodeQueries::from_uri(&parts.uri, false) - .unwrap_or_else(|| DecodeQueries(HashMap::new()))) - } -} - -impl DecodeQueries { - pub fn from_uri(uri: &Uri, is_http_trigger: bool) -> Option { - let query = uri.query(); - if query.is_none() { - return None; - } - let query = query.unwrap(); - if is_http_trigger { + if include_all_query { + let queries = + serde_urlencoded::from_str::>(&query).unwrap_or_default(); + queries + .into_iter() + .map(|(k, v)| (k, to_raw_value(&v))) + .collect() + } else { + let parse_query_args = include_query + .map(|s| s.split(",").map(|p| p.to_string()).collect::>()) + .unwrap_or_default(); + let mut args = HashMap::new(); + if !parse_query_args.is_empty() { let queries = - serde_urlencoded::from_str::>(query).unwrap_or_default(); - Some(DecodeQueries( - queries - .into_iter() - .map(|(k, v)| (k, to_raw_value(&v))) - .collect(), - )) - } else { - let include_query = serde_urlencoded::from_str::(query) - .map(|x| x.include_query) - .ok() - .flatten() - .unwrap_or_default(); - let parse_query_args = include_query - .split(",") - .map(|s| s.to_string()) - .collect::>(); - let mut args = HashMap::new(); - if !parse_query_args.is_empty() { - let queries = serde_urlencoded::from_str::>(query) - .unwrap_or_default(); - parse_query_args.iter().for_each(|h| { - if let Some(v) = queries.get(h) { - args.insert(h.to_string(), to_raw_value(v)); - } - }); - } - Some(DecodeQueries(args)) + serde_urlencoded::from_str::>(&query).unwrap_or_default(); + parse_query_args.iter().for_each(|h| { + if let Some(v) = queries.get(h) { + args.insert(h.to_string(), to_raw_value(v)); + } + }); } + args } } -// impl<'c> PushArgs<'c> { -// pub fn insert, V: Into>>(&mut self, k: K, v: V) { -// self.extra.insert(k.into(), v.into()); -// } -// } - fn restructure_cloudevents_metadata( mut p: HashMap>, ) -> Result>, Error> { diff --git a/backend/windmill-api/src/auth.rs b/backend/windmill-api/src/auth.rs index 5f66a0deaf..0765f701f9 100644 --- a/backend/windmill-api/src/auth.rs +++ b/backend/windmill-api/src/auth.rs @@ -121,11 +121,10 @@ impl AuthCache { match jwt_result { Ok(claims) => { - if w_id.is_some_and(|w_id| w_id != claims.workspace_id) { + if w_id.is_some_and(|w_id| !claims.allowed_in_workspace(&w_id)) { tracing::error!("JWT auth error: workspace_id mismatch"); return None; } - let username_override = username_override_from_label(claims.label); let authed = crate::db::ApiAuthed { email: claims.email, @@ -415,6 +414,7 @@ pub struct Tokened { pub token: String, } +#[derive(Clone, Debug)] pub struct OptTokened { #[allow(dead_code)] pub token: Option, diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 1231bcf042..bcc287ec04 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -46,10 +46,10 @@ use serde::de::DeserializeOwned; use windmill_common::error::Error; #[cfg(all(feature = "enterprise", feature = "kafka", feature = "private"))] -use crate::kafka_triggers_ee::KafkaTriggerConfigConnection; +use crate::triggers::kafka::KafkaTriggerConfigConnection; #[cfg(feature = "mqtt_trigger")] -use crate::mqtt_triggers::{MqttClientVersion, MqttV3Config, MqttV5Config, SubscribeTopic}; +use crate::triggers::mqtt::{MqttClientVersion, MqttV3Config, MqttV5Config, SubscribeTopic}; #[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] use crate::triggers::nats::NatsTriggerConfigConnection; @@ -183,6 +183,7 @@ pub struct GcpTriggerConfig { pub create_update: Option, pub topic_id: String, pub auto_acknowledge_msg: Option, + pub ack_deadline: Option, } #[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] @@ -397,6 +398,7 @@ async fn set_gcp_trigger_config( gcp_config.create_update, false, capture_config.is_flow, + gcp_config.ack_deadline, ) .await?; gcp_config.create_update = Some(config); @@ -964,6 +966,8 @@ async fn http_payload( Path((w_id, runnable_kind, path, route_path)): Path<(String, RunnableKind, String, StripPath)>, args: RawHttpTriggerArgs, ) -> std::result::Result { + use crate::args::{build_headers, build_query}; + let path = path.replace(".", "/"); let is_flow = matches!(runnable_kind, RunnableKind::Flow); let route_path = route_path.to_path(); @@ -1000,9 +1004,18 @@ async fn http_payload( .map(|(k, v)| (k.to_string(), v.to_string())) .collect(); + let headers = build_headers(&args.0.metadata.headers, None, true); + let query = build_query(args.0.metadata.query.as_deref(), None, true); + let preprocessor_args = args .clone() - .to_v2_preprocessor_args(&http_trigger_config.route_path, &route_path, ¶ms) + .to_v2_preprocessor_args( + &http_trigger_config.route_path, + &route_path, + ¶ms, + headers, + query, + ) .map_err(|e| e.into_response())?; let main_args = args diff --git a/backend/windmill-api/src/concurrency_groups.rs b/backend/windmill-api/src/concurrency_groups.rs index ba85043a73..bfd0b8182f 100644 --- a/backend/windmill-api/src/concurrency_groups.rs +++ b/backend/windmill-api/src/concurrency_groups.rs @@ -159,24 +159,24 @@ async fn get_concurrent_intervals( let lq = ListCompletedQuery { order_desc: Some(true), ..lq }; let lqc = lq.clone(); let lqq: ListQueueQuery = lqc.into(); - let mut sqlb_q = SqlBuilder::select_from("v2_as_queue") + let mut sqlb_q = SqlBuilder::select_from("v2_job_queue") .fields(UnifiedJob::queued_job_fields()) .order_by("created_at", lq.order_desc.unwrap_or(true)) .limit(row_limit) .clone(); - let mut sqlb_c = SqlBuilder::select_from("v2_as_completed_job") + let mut sqlb_c = SqlBuilder::select_from("v2_job_completed") .fields(UnifiedJob::completed_job_fields()) - .order_by("started_at", lq.order_desc.unwrap_or(true)) + .order_by("completed_at", lq.order_desc.unwrap_or(true)) .limit(row_limit) .clone(); - let mut sqlb_q_user = SqlBuilder::select_from("v2_as_queue") + let mut sqlb_q_user = SqlBuilder::select_from("v2_job_queue") .fields(&["id"]) .order_by("created_at", lq.order_desc.unwrap_or(true)) .limit(row_limit) .clone(); - let mut sqlb_c_user = SqlBuilder::select_from("v2_as_completed_job") + let mut sqlb_c_user = SqlBuilder::select_from("v2_job_completed") .fields(&["id"]) - .order_by("started_at", lq.order_desc.unwrap_or(true)) + .order_by("completed_at", lq.order_desc.unwrap_or(true)) .limit(row_limit) .clone(); @@ -209,6 +209,10 @@ async fn get_concurrent_intervals( started_after: _, created_before: _, created_after: _, + created_before_queue: _, + created_after_queue: _, + completed_after: _, + completed_before: _, created_or_started_before: _, created_or_started_after: _, created_or_started_after_completed_jobs: _, @@ -296,6 +300,7 @@ async fn get_concurrent_intervals( duration_ms: j.duration_ms, }) .collect(); + let jobs = running_jobs_db .into_iter() .filter(|j| running_jobs_user.iter().any(|id| j.id == *id)) @@ -306,6 +311,7 @@ async fn get_concurrent_intervals( ) .map(From::from) .collect(); + Ok(Json(ExtendedJobs { jobs, obscured_jobs, diff --git a/backend/windmill-api/src/configs.rs b/backend/windmill-api/src/configs.rs index 3d7c47f89e..719437b29c 100644 --- a/backend/windmill-api/src/configs.rs +++ b/backend/windmill-api/src/configs.rs @@ -7,7 +7,7 @@ */ use axum::{ - extract::{Extension, Path}, + extract::{Extension, Path, Query}, routing::{get, post}, Json, Router, }; @@ -18,6 +18,7 @@ use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::{ error::{self}, + utils::Pagination, worker::MIN_PERIODIC_SCRIPT_INTERVAL_SECONDS, DB, }; @@ -172,7 +173,7 @@ async fn update_config( let mut tx = db.begin().await?; sqlx::query!( - "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2", + "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = EXCLUDED.config", &name, config ) @@ -239,11 +240,19 @@ struct AutoscalingEvent { async fn list_autoscaling_events( Extension(db): Extension, Path(worker_group): Path, + Query(mut pagination): Query, ) -> error::JsonResult> { + if pagination.per_page.is_none() { + pagination.per_page = Some(5); + } + let (per_page, offset) = windmill_common::utils::paginate(pagination); + let events = sqlx::query_as!( AutoscalingEvent, - "SELECT id, worker_group, event_type::text, desired_workers, reason, applied_at FROM autoscaling_event WHERE worker_group = $1 ORDER BY applied_at DESC LIMIT 5", - worker_group + "SELECT id, worker_group, event_type::text, desired_workers, reason, applied_at FROM autoscaling_event WHERE worker_group = $1 ORDER BY applied_at DESC LIMIT $2 OFFSET $3", + worker_group, + per_page as i64, + offset as i64 ) .fetch_all(&db) .await?; @@ -263,8 +272,7 @@ async fn native_kubernetes_autoscaling_healthcheck( } #[cfg(not(all(feature = "enterprise", feature = "private")))] -async fn native_kubernetes_autoscaling_healthcheck( -) -> Result<(), error::Error> { +async fn native_kubernetes_autoscaling_healthcheck() -> Result<(), error::Error> { Err(error::Error::BadRequest( "Native Kubernetes autoscaling available only in the enterprise version".to_string(), )) diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index e5d8c433f1..c5d16ebffe 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -101,14 +101,20 @@ impl Migrate for CustomMigrator { let mut r = false; while !r { - r = sqlx::query_scalar!("SELECT pg_try_advisory_lock($1)", lock_id) - .fetch_one(&mut *self.inner) + r = match tokio::time::timeout(std::time::Duration::from_secs(5), sqlx::query_scalar!("SELECT pg_try_advisory_lock($1)", lock_id) + .fetch_one(&mut *self.inner)) .await - .map_err(|e| { - tracing::error!("Error acquiring lock: {e:#}"); - sqlx::migrate::MigrateError::Execute(e) - })? - .unwrap_or(false); + { + Ok(Ok(r)) => r.unwrap_or(false), + Ok(Err(e)) => { + tracing::error!("Error acquiring lock: {e:#}"); + return Err(sqlx::migrate::MigrateError::Execute(e)); + } + Err(e) => { + tracing::error!("Timed out acquiring lock retrying in 5s: {e:#}"); + false + } + }; if !r { tracing::info!("PG migration lock already acquired by another server or worker, a migration is in progress, this may take a long time if you have many jobs and be normal, rechecking in 5s."); tokio::time::sleep(std::time::Duration::from_secs(5)).await; @@ -196,14 +202,17 @@ impl Migrate for CustomMigrator { } } -pub async fn migrate(db: &DB) -> Result>, Error> { +pub async fn migrate( + db: &DB, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) -> Result>, Error> { let migrator = db.acquire().await?; let mut custom_migrator = CustomMigrator { inner: migrator }; if let Err(err) = sqlx::query!( "DELETE FROM _sqlx_migrations WHERE version=20250131115248 OR version=20250902085503 OR version=20250201145630 OR - version=20250201145631 OR version=20250201145632" + version=20250201145631 OR version=20250201145632 OR version=20251006143821" ) .execute(db) .await @@ -211,20 +220,27 @@ pub async fn migrate(db: &DB) -> Result>, Error> { tracing::info!("Could not remove sqlx migrations: {err:#}"); } - match sqlx::migrate!("../migrations") - .run_direct(&mut custom_migrator) - .await - { - Ok(_) => Ok(()), - Err(sqlx::migrate::MigrateError::VersionMissing(e)) => { - tracing::error!("Database had been applied more migrations than this container. - This usually mean than another container on a more recent version migrated the database and this one is on an earlier version. - Please update the container to latest. Not critical, but may cause issues if migration introduced a breaking change. Version missing: {e:#}"); - custom_migrator.unlock().await?; - Ok(()) + tokio::select! { + _ = killpill_rx.recv() => { + tracing::info!("Killpill received, stopping migration"); + return Ok(None); } - Err(err) => Err(err), - }?; + migration_result = sqlx::migrate!("../migrations") + .run_direct(&mut custom_migrator) + => { + match migration_result { + Ok(_) => Ok(()), + Err(sqlx::migrate::MigrateError::VersionMissing(e)) => { + tracing::error!("Database had been applied more migrations than this container. + This usually mean than another container on a more recent version migrated the database and this one is on an earlier version. + Please update the container to latest. Not critical, but may cause issues if migration introduced a breaking change. Version missing: {e:#}"); + custom_migrator.unlock().await?; + Ok(()) + } + Err(err) => Err(err), + }?; + } + } return crate::live_migrations::custom_migrations(&mut custom_migrator, db).await; } diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index 02d3b3c4fb..41e8d4709d 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -79,7 +79,7 @@ async fn create_draft( "INSERT INTO draft (workspace_id, path, value, typ) VALUES ($1, $2, $3::text::json, $4) - ON CONFLICT (workspace_id, path, typ) DO UPDATE SET value = $3::text::json", + ON CONFLICT (workspace_id, path, typ) DO UPDATE SET value = EXCLUDED.value", &w_id, draft.path, //to preserve key orders diff --git a/backend/windmill-api/src/ee_oss.rs b/backend/windmill-api/src/ee_oss.rs index cec4e4db24..a69c59c573 100644 --- a/backend/windmill-api/src/ee_oss.rs +++ b/backend/windmill-api/src/ee_oss.rs @@ -11,10 +11,7 @@ use {crate::db::ApiAuthed, windmill_common::DB}; #[cfg(not(feature = "private"))] use anyhow::anyhow; #[cfg(all(feature = "enterprise", not(feature = "private")))] -use std::sync::Arc; -#[cfg(all(feature = "enterprise", not(feature = "private")))] -use tokio::sync::RwLock; - +use {std::sync::Arc, tokio::sync::RwLock}; #[cfg(not(feature = "private"))] pub async fn validate_license_key(_license_key: String) -> anyhow::Result<(String, bool)> { // Implementation is not open source diff --git a/backend/windmill-api/src/embeddings.rs b/backend/windmill-api/src/embeddings.rs index 9b3d21844a..875bcf683a 100644 --- a/backend/windmill-api/src/embeddings.rs +++ b/backend/windmill-api/src/embeddings.rs @@ -441,7 +441,7 @@ impl EmbeddingsDb { &query_embedding, limit.unwrap_or(10) as usize, Some(&filter), - Some(0.75), + Some(0.8), ); let results: Result> = results @@ -480,7 +480,17 @@ impl EmbeddingsDb { }) .collect(); - results + let mut results = results?; + + if results.len() > 1 { + let top_score = results[0].score; + results = results + .into_iter() + .take_while(|r| (top_score - r.score) / top_score <= 0.05) + .collect(); + } + + Ok(results) } pub async fn query_resource_types( diff --git a/backend/windmill-api/src/flow_conversations.rs b/backend/windmill-api/src/flow_conversations.rs new file mode 100644 index 0000000000..6f046c4738 --- /dev/null +++ b/backend/windmill-api/src/flow_conversations.rs @@ -0,0 +1,253 @@ +use axum::{ + extract::{Path, Query}, + routing::{delete, get}, + Extension, Json, Router, +}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sql_builder::prelude::*; +use sqlx::{FromRow, Postgres}; +use uuid::Uuid; + +use crate::db::ApiAuthed; +use windmill_common::{ + db::UserDB, + error::{JsonResult, Result}, + flow_conversations::MessageType, + utils::{not_found_if_none, paginate, Pagination}, +}; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list_conversations)) + .route("/delete/:conversation_id", delete(delete_conversation)) + .route("/:conversation_id/messages", get(list_messages)) +} + +#[derive(Serialize, FromRow, Debug)] +pub struct FlowConversation { + pub id: Uuid, + pub workspace_id: String, + pub flow_path: String, + pub title: Option, + pub created_at: DateTime, + pub updated_at: DateTime, + pub created_by: String, +} + +#[derive(Serialize, FromRow, Debug)] +pub struct FlowConversationMessage { + pub id: Uuid, + pub conversation_id: Uuid, + pub message_type: MessageType, + pub content: String, + pub job_id: Option, + pub created_at: DateTime, + pub step_name: Option, + pub success: bool, +} + +#[derive(Deserialize)] +pub struct ListConversationsQuery { + pub flow_path: Option, + pub after_id: Option, +} + +async fn list_conversations( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(pagination): Query, + Query(query): Query, +) -> JsonResult> { + let (per_page, offset) = paginate(pagination); + let mut tx = user_db.clone().begin(&authed).await?; + + let mut sqlb = SqlBuilder::select_from("flow_conversation"); + sqlb.fields(&[ + "id", + "workspace_id", + "flow_path", + "title", + "created_at", + "updated_at", + "created_by", + ]) + .and_where_eq("workspace_id", "?".bind(&w_id)); + + if let Some(flow_path) = &query.flow_path { + sqlb.and_where_eq("flow_path", "?".bind(flow_path)); + } + if let Some(after_id) = &query.after_id { + let message_id_created_at = sqlx::query_scalar!( + "SELECT created_at FROM flow_conversation_message WHERE id = $1", + after_id + ) + .fetch_one(&mut *tx) + .await?; + sqlb.and_where_gt("created_at", "?".bind(&message_id_created_at.to_rfc3339())); + } + + sqlb.order_by("updated_at", true) + .limit(per_page as i64) + .offset(offset as i64); + + let sql = sqlb.sql().map_err(|e| { + windmill_common::error::Error::internal_err(format!("Failed to build SQL: {}", e)) + })?; + + let conversations = sqlx::query_as::(&sql) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + Ok(Json(conversations)) +} + +pub async fn get_or_create_conversation_with_id( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + flow_path: &str, + username: &str, + title: &str, + conversation_id: Uuid, +) -> Result { + // Check if conversation already exists + let existing_conversation = sqlx::query_as!( + FlowConversation, + "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by + FROM flow_conversation + WHERE id = $1 AND workspace_id = $2", + conversation_id, + w_id + ) + .fetch_optional(&mut **tx) + .await?; + + if let Some(existing) = existing_conversation { + return Ok(existing); + } + + // Truncate title to 25 char characters max + let title = if title.len() > 25 { + format!("{}...", &title[..25]) + } else { + title.to_string() + }; + // Create new conversation with provided ID + let conversation = sqlx::query_as!( + FlowConversation, + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title) + VALUES ($1, $2, $3, $4, $5) + RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by", + conversation_id, + w_id, + flow_path, + username, + title + ) + .fetch_one(&mut **tx) + .await?; + Ok(conversation) +} + +async fn delete_conversation( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, conversation_id)): Path<(String, Uuid)>, +) -> Result { + let mut tx = user_db.clone().begin(&authed).await?; + + // Verify the conversation exists and belongs to the user + let conversation = sqlx::query_as!( + FlowConversation, + "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by + FROM flow_conversation + WHERE id = $1 AND workspace_id = $2", + conversation_id, + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + + not_found_if_none(conversation, "Conversation", conversation_id.to_string())?; + + // Delete the conversation (messages will be cascade deleted) + sqlx::query!( + "DELETE FROM flow_conversation WHERE id = $1 AND workspace_id = $2", + conversation_id, + &w_id + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + // Delete associated memory in background (non-blocking cleanup) + let w_id_clone = w_id.clone(); + tokio::spawn(async move { + if let Err(e) = + windmill_worker::memory_oss::delete_conversation_memory(&w_id_clone, conversation_id) + .await + { + tracing::error!( + "Failed to delete memory for conversation {} in workspace {}: {:?}", + conversation_id, + w_id_clone, + e + ); + } + }); + + Ok(format!("Conversation {} deleted", conversation_id)) +} + +async fn list_messages( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, conversation_id)): Path<(String, Uuid)>, + Query(pagination): Query, +) -> JsonResult> { + let (per_page, offset) = paginate(pagination); + let mut tx = user_db.clone().begin(&authed).await?; + + // Verify the conversation exists and belongs to the user + let conversation_exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM flow_conversation WHERE id = $1 AND workspace_id = $2)", + conversation_id, + &w_id + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + + if !conversation_exists { + return Err(windmill_common::error::Error::NotFound(format!( + "Conversation not found or access denied: {}", + conversation_id + ))); + } + + // Fetch messages for this conversation, oldest first, but reverse the order of the messages for easy rendering on the frontend + let messages = sqlx::query_as!( + FlowConversationMessage, + r#"SELECT id, conversation_id, message_type as "message_type: MessageType", content, job_id, created_at, step_name, success + FROM ( + SELECT id, conversation_id, message_type, content, job_id, created_at, step_name, success + FROM flow_conversation_message + WHERE conversation_id = $1 + ORDER BY created_at DESC, CASE WHEN message_type = 'user' THEN 0 ELSE 1 END + LIMIT $2 OFFSET $3 + ) AS messages + ORDER BY created_at ASC, CASE WHEN message_type = 'user' THEN 0 ELSE 1 END + "#, + conversation_id, + per_page as i64, + offset as i64 + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + Ok(Json(messages)) +} diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 49a143fb98..0f4e8db038 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -32,8 +32,8 @@ use sql_builder::prelude::*; use sqlx::{FromRow, Postgres, Transaction}; use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; -use windmill_common::utils::query_elems_from_hub; -use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; +use windmill_common::utils::{query_elems_from_hub, WarnAfterExt}; +use windmill_common::worker::{to_raw_value, CLOUD_HOSTED, MIN_VERSION_SUPPORTS_DEBOUNCING}; use windmill_common::HUB_BASE_URL; use windmill_common::{ db::UserDB, @@ -46,6 +46,7 @@ use windmill_common::{ }; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; use windmill_queue::{push, schedule::push_scheduled_job, PushIsolationLevel}; +use windmill_worker::scoped_dependency_map::ScopedDependencyMap; pub fn workspaced_service() -> Router { Router::new() @@ -279,33 +280,48 @@ async fn toggle_workspace_error_handler( let mut tx = user_db.begin(&authed).await?; let error_handler_maybe: Option = sqlx::query_scalar!( - "SELECT error_handler FROM workspace_settings WHERE workspace_id = $1", + r#" + SELECT + error_handler + FROM + workspace_settings + WHERE + workspace_id = $1 + "#, w_id ) .fetch_optional(&mut *tx) .await? .unwrap_or(None); - return match error_handler_maybe { + let response = match error_handler_maybe { Some(_) => { sqlx::query_scalar!( - "UPDATE flow SET ws_error_handler_muted = $3 WHERE path = $1 AND workspace_id = $2", + r#" + UPDATE + flow + SET + ws_error_handler_muted = $3 + WHERE + path = $1 AND + workspace_id = $2 + "#, path.to_path(), w_id, req.muted, ) .execute(&mut *tx) .await?; - tx.commit().await?; Ok("".to_string()) } - None => { - tx.commit().await?; - Err(Error::ExecutionErr( - "Workspace error handler needs to be defined".to_string(), - )) - } + None => Err(Error::BadRequest( + "Workspace error handler needs to be defined".to_string(), + )), }; + + tx.commit().await?; + + return response; } async fn check_path_conflict<'c>( @@ -376,6 +392,20 @@ async fn list_paths_from_workspace_runnable( Ok(Json(runnables)) } +async fn validate_flow(new_flow: &NewFlow) -> error::Result<()> { + #[cfg(not(feature = "enterprise"))] + if new_flow.ws_error_handler_muted.is_some_and(|val| val) { + return Err(Error::BadRequest( + "Muting the error handler for certain flow is only available in enterprise version" + .to_string(), + )); + } + + guard_flow_from_debounce_data(new_flow).await?; + + return Ok(()); +} + async fn create_flow( authed: ApiAuthed, Extension(db): Extension, @@ -385,6 +415,7 @@ async fn create_flow( Json(nf): Json, ) -> Result<(StatusCode, String)> { check_scopes(&authed, || format!("flows:write:{}", nf.path))?; + validate_flow(&nf).await?; if *CLOUD_HOSTED { let nb_flows = sqlx::query_scalar!("SELECT COUNT(*) FROM flow WHERE workspace_id = $1", &w_id) @@ -411,18 +442,6 @@ async fn create_flow( )); } } - #[cfg(not(feature = "enterprise"))] - if nf - .value - .get("ws_error_handler_muted") - .map(|val| val.as_bool().unwrap_or(false)) - .is_some_and(|val| val) - { - return Err(Error::BadRequest( - "Muting the error handler for certain flow is only available in enterprise version" - .to_string(), - )); - } // cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?; let authed = maybe_refresh_folders(&nf.path, &w_id, authed, &db).await; @@ -448,17 +467,13 @@ async fn create_flow( w_id, nf.path, nf.summary, - nf.description.unwrap_or_else(String::new), + nf.description.as_deref().unwrap_or(""), nf.draft_only, nf.tag, nf.dedicated_worker, nf.visible_to_runner_only.unwrap_or(false), - if nf.on_behalf_of_email.is_some() { - Some(&authed.email) - } else { - None - }, - nf.value, + nf.on_behalf_of_email.and(Some(&authed.email)), + sqlx::types::Json(&nf.value) as _, schema_str, &authed.username, ) @@ -471,7 +486,7 @@ async fn create_flow( RETURNING id", w_id, nf.path, - nf.value, + sqlx::types::Json(nf.value) as _, schema_str, &authed.username, ) @@ -545,6 +560,8 @@ async fn create_flow( None, Some(&authed.clone().into()), false, + None, + None, ) .await?; @@ -721,7 +738,7 @@ async fn update_flow_history( } sqlx::query!( - "INSERT INTO deployment_metadata (workspace_id, path, flow_version, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL DO UPDATE SET deployment_msg = $4", + "INSERT INTO deployment_metadata (workspace_id, path, flow_version, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL DO UPDATE SET deployment_msg = EXCLUDED.deployment_msg", w_id, path_o.unwrap(), version, @@ -743,19 +760,7 @@ async fn update_flow( ) -> Result { let flow_path = flow_path.to_path(); check_scopes(&authed, || format!("flows:write:{}", flow_path))?; - - #[cfg(not(feature = "enterprise"))] - if nf - .value - .get("ws_error_handler_muted") - .map(|val| val.as_bool().unwrap_or(false)) - .is_some_and(|val| val) - { - return Err(Error::BadRequest( - "Muting the error handler for certain flow is only available in enterprise version" - .to_string(), - )); - } + validate_flow(&nf).await?; let authed = maybe_refresh_folders(&flow_path, &w_id, authed, &db).await; let mut tx = user_db.clone().begin(&authed).await?; @@ -798,16 +803,12 @@ async fn update_flow( path = $11 AND workspace_id = $12", if is_new_path { flow_path } else { &nf.path }, nf.summary, - nf.description.unwrap_or_else(String::new), + nf.description.as_deref().unwrap_or(""), nf.tag, nf.dedicated_worker, nf.visible_to_runner_only.unwrap_or(false), - if nf.on_behalf_of_email.is_some() { - Some(&authed.email) - } else { - None - }, - nf.value, + nf.on_behalf_of_email.and(Some(&authed.email)), + sqlx::types::Json(&nf.value) as _, schema_str, authed.username, flow_path, @@ -883,11 +884,20 @@ async fn update_flow( .await?; } + // Row lock debounce key for path. We need this to make all updates of runnables sequential and predictable. + tokio::time::timeout( + core::time::Duration::from_secs(60), + windmill_common::jobs::lock_debounce_key(&w_id, &nf.path, &mut tx), + ) + .warn_after_seconds(10) + .await??; + + // This will lock anyone who is trying to iterate on flow_versions with given path and parameters. let version = sqlx::query_scalar!( "INSERT INTO flow_version (workspace_id, path, value, schema, created_by) VALUES ($1, $2, $3, $4::text::json, $5) RETURNING id", w_id, nf.path, - nf.value, + sqlx::types::Json(nf.value) as _, schema_str, &authed.username, ) @@ -899,6 +909,7 @@ async fn update_flow( )) })?; + // TODO: This should happen only after we are done with dependency job. sqlx::query!( "UPDATE flow SET versions = array_append(versions, $1) WHERE path = $2 AND workspace_id = $3", version, nf.path, w_id @@ -937,7 +948,7 @@ async fn update_flow( clear_schedule(&mut tx, &schedule.path, &w_id).await?; if schedule.enabled { - tx = push_scheduled_job(&db, tx, &schedule, None).await?; + tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; } } @@ -1012,8 +1023,11 @@ async fn update_flow( None, Some(&authed.clone().into()), false, + None, + None, ) .await?; + sqlx::query!( "UPDATE flow SET dependency_job = $1 WHERE path = $2 AND workspace_id = $3", dependency_job_uuid, @@ -1311,7 +1325,11 @@ async fn archive_flow_by_path( Some([("workspace", w_id.as_str())].into()), ) .await?; - tx.commit().await?; + + ScopedDependencyMap::clear_map_for_item(path, &w_id, "flow", tx, &None) + .await + .commit() + .await?; handle_deployment_metadata( &authed.email, @@ -1344,6 +1362,22 @@ async fn archive_flow_by_path( Ok(format!("Flow {path} archived")) } +/// Validates that flow debouncing configuration is supported by all workers +/// Returns an error if debouncing is configured but workers are behind required version +async fn guard_flow_from_debounce_data(nf: &NewFlow) -> Result<()> { + if !*MIN_VERSION_SUPPORTS_DEBOUNCING.read().await && { + let flow_value = nf.parse_flow_value()?; + flow_value.debounce_key.is_some() || flow_value.debounce_delay_s.is_some() + } { + tracing::warn!( + "Flow debouncing configuration rejected: workers are behind minimum required version for debouncing feature" + ); + Err(Error::WorkersAreBehind { feature: "Debouncing".into(), min_version: "1.566.0".into() }) + } else { + Ok(()) + } +} + #[derive(Deserialize)] struct DeleteFlowQuery { keep_captures: Option, @@ -1476,6 +1510,7 @@ mod tests { hash: None, tag_override: None, is_trigger: None, + pass_flow_input_directly: None, }), stop_after_if: None, stop_after_all_iters_if: None, @@ -1491,6 +1526,7 @@ mod tests { continue_on_error: None, skip_if: None, apply_preprocessor: None, + pass_flow_input_directly: None, }, FlowModule { id: "b".to_string(), @@ -1524,6 +1560,7 @@ mod tests { continue_on_error: None, skip_if: None, apply_preprocessor: None, + pass_flow_input_directly: None, }, FlowModule { id: "c".to_string(), @@ -1554,6 +1591,7 @@ mod tests { continue_on_error: None, skip_if: None, apply_preprocessor: None, + pass_flow_input_directly: None, }, ], failure_module: Some(Box::new(FlowModule { @@ -1564,6 +1602,7 @@ mod tests { hash: None, tag_override: None, is_trigger: None, + pass_flow_input_directly: None, } .into(), stop_after_if: Some(StopAfterIf { @@ -1583,6 +1622,7 @@ mod tests { continue_on_error: None, skip_if: None, apply_preprocessor: None, + pass_flow_input_directly: None, })), preprocessor_module: None, same_worker: false, @@ -1593,6 +1633,9 @@ mod tests { priority: None, early_return: None, concurrency_key: None, + chat_input_enabled: None, + debounce_key: None, + debounce_delay_s: None, }; let expect = serde_json::json!({ "modules": [ diff --git a/backend/windmill-api/src/folders.rs b/backend/windmill-api/src/folders.rs index db9fa72c91..9cf3c9af2b 100644 --- a/backend/windmill-api/src/folders.rs +++ b/backend/windmill-api/src/folders.rs @@ -202,6 +202,7 @@ async fn create_folder( )); } + if let Err(e) = sqlx::query_as!( Folder, "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, now())", @@ -214,7 +215,38 @@ async fn create_folder( authed.username ) .execute(&mut *tx) - .await?; + .await { + let exists_for_user = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM folder WHERE name = $1 AND workspace_id = $2 AND $3 = ANY(owners))", + ng.name, + w_id, + authed.username + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM folder WHERE name = $1 AND workspace_id = $2)", + ng.name, + w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + if !exists_for_user && exists { + return Err(windmill_common::error::Error::BadRequest(format!( + "Folder '{}' already exists in workspace '{}' but you do not have permission to read to it", ng.name, w_id + ))); + } else if exists { + return Err(windmill_common::error::Error::BadRequest(format!( + "Folder '{}' already exists in workspace '{}'", ng.name, w_id + ))); + } else { + return Err(windmill_common::error::Error::InternalErr(format!( + "Failed to create folder: {}", e + ))); + } + } audit_log( &mut *tx, @@ -329,6 +361,11 @@ async fn update_folder( ); } if let Some(extra_perms) = ng.extra_perms { + if !extra_perms.is_object() { + return Err(windmill_common::error::Error::BadRequest(format!( + "extra_perms must be an object, received {}", extra_perms.to_string() + ))); + } sqlb.set( "extra_perms", "?".bind(&serde_json::to_string(&extra_perms).map_err(to_anyhow)?), diff --git a/backend/windmill-api/src/gcp_triggers_oss.rs b/backend/windmill-api/src/gcp_triggers_oss.rs deleted file mode 100644 index ce2f81f3c9..0000000000 --- a/backend/windmill-api/src/gcp_triggers_oss.rs +++ /dev/null @@ -1,120 +0,0 @@ -#[cfg(feature = "private")] -#[allow(unused)] -pub use crate::gcp_triggers_ee::*; - -#[cfg(not(feature = "private"))] -use { - crate::{db::DB, trigger_helpers::TriggerJobArgs}, - serde::{Deserialize, Serialize}, - serde_json::value::RawValue, - sqlx::FromRow, - std::collections::HashMap, - windmill_common::{triggers::TriggerKind, utils::empty_as_none, worker::to_raw_value}, -}; - -#[cfg(not(feature = "private"))] -type SqlxJson = sqlx::types::Json; - -#[derive(sqlx::Type, Debug, Deserialize, Serialize)] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -#[sqlx(type_name = "DELIVERY_MODE", rename_all = "lowercase")] -#[allow(unused)] -#[cfg(not(feature = "private"))] -pub enum DeliveryType { - Pull, - Push, -} - -#[cfg(not(feature = "private"))] -impl Default for DeliveryType { - fn default() -> Self { - Self::Pull - } -} - -#[derive(FromRow, Deserialize, Serialize, Debug)] -#[allow(unused)] -#[cfg(not(feature = "private"))] -pub struct PushConfig { - #[serde(deserialize_with = "empty_as_none")] - route_path: Option, - #[serde(deserialize_with = "empty_as_none")] - audience: Option, - authenticate: bool, - base_endpoint: String, -} -#[derive(Default, Debug, Serialize, Deserialize)] -#[allow(unused)] -#[cfg(not(feature = "private"))] -pub struct CreateUpdateConfig { - pub delivery_type: DeliveryType, - #[serde(default, deserialize_with = "empty_as_none")] - pub subscription_id: Option, - pub delivery_config: Option>, -} - -#[derive(Debug, Deserialize, Serialize)] -#[cfg(not(feature = "private"))] -pub struct ExistingGcpSubscription { - pub subscription_id: String, - pub base_endpoint: String, -} - -#[derive(Debug, Deserialize, Serialize, sqlx::Type)] -#[serde(rename_all = "snake_case")] -#[sqlx(type_name = "GCP_SUBSCRIPTION_MODE", rename_all = "snake_case")] -#[cfg(not(feature = "private"))] -pub enum SubscriptionMode { - Existing, - CreateUpdate, -} - -#[cfg(not(feature = "private"))] -pub fn start_consuming_gcp_pubsub_event( - _db: DB, - mut _killpill_rx: tokio::sync::broadcast::Receiver<()>, -) -> () { - // implementation is not open source -} - -#[cfg(not(feature = "private"))] -#[derive(FromRow, Deserialize, Serialize, Debug)] -pub struct GcpTrigger { - pub gcp_resource_path: String, - pub subscription_id: String, - pub delivery_type: DeliveryType, - pub delivery_config: Option>, - pub subscription_mode: SubscriptionMode, - pub topic_id: String, - pub path: String, - pub script_path: String, - pub is_flow: bool, - pub workspace_id: String, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - pub extra_perms: Option, - pub error: Option, - pub server_id: Option, - pub last_server_ping: Option>, - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub auto_acknowledge_msg: Option, -} - -#[cfg(not(feature = "private"))] -impl TriggerJobArgs for GcpTrigger { - fn v1_payload_fn(payload: String) -> HashMap> { - HashMap::from([("payload".to_string(), to_raw_value(&payload))]) - } - - fn trigger_kind() -> TriggerKind { - TriggerKind::Gcp - } -} diff --git a/backend/windmill-api/src/groups.rs b/backend/windmill-api/src/groups.rs index 4c526ffe26..4325a7c9c5 100644 --- a/backend/windmill-api/src/groups.rs +++ b/backend/windmill-api/src/groups.rs @@ -286,13 +286,17 @@ async fn create_igroup( Extension(db): Extension, Json(ng): Json, ) -> Result { + use uuid::Uuid; + require_super_admin(&db, &authed.email).await?; let mut tx = db.begin().await?; + let id = Uuid::new_v4().to_string(); sqlx::query!( - "INSERT INTO instance_group (name, summary) VALUES ($1, $2) ON CONFLICT DO NOTHING", + "INSERT INTO instance_group (name, summary, id) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", ng.name, ng.summary, + id, ) .execute(&mut *tx) .await?; @@ -629,6 +633,20 @@ async fn add_user_igroup( Some([("email", email.as_str())].into()), ) .await?; + + // Sync user to workspaces configured with this instance group + #[cfg(all(feature = "private", feature = "enterprise"))] + { + use crate::workspaces_ee::auto_add_user; + let workspaces = sqlx::query!("SELECT workspace_id, auto_add_instance_groups_roles FROM workspace_settings WHERE $1 = ANY(COALESCE(auto_add_instance_groups, '{}'))", &name).fetch_all(&mut *tx).await?; + for ws in workspaces { + let role = ws.auto_add_instance_groups_roles.and_then(|r| r.get(&name).and_then(|v| v.as_str().map(String::from))).unwrap_or_else(|| "developer".to_string()); + let (is_admin, is_operator) = match role.as_str() { "admin" => (true, false), "operator" => (false, true), _ => (false, false) }; + auto_add_user(&email, &ws.workspace_id, &is_operator, &mut tx, &authed, Some(serde_json::json!({"source": "instance_group", "group": &name}))).await?; + if is_admin { sqlx::query!("UPDATE usr SET is_admin = true WHERE workspace_id = $1 AND email = $2", &ws.workspace_id, &email).execute(&mut *tx).await?; } + } + } + tx.commit().await?; Ok(format!("Added {} to igroup {}", email, name)) } @@ -777,8 +795,16 @@ async fn remove_user_igroup( Some([("email", email.as_str())].into()), ) .await?; + + // Remove user from workspaces where they were added via this instance group + #[cfg(all(feature = "private", feature = "enterprise"))] + { + use crate::workspaces_ee::remove_users_from_instance_group_workspaces; + remove_users_from_instance_group_workspaces(&email, &name, &mut tx).await?; + } + tx.commit().await?; - Ok(format!("Added {} to igroup {}", email, name)) + Ok(format!("Removed {} from igroup {}", email, name)) } async fn remove_user( diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index 61c9f0606b..defb53db76 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -5,8 +5,6 @@ pub use crate::job_helpers_ee::*; #[cfg(not(feature = "private"))] use axum::Router; #[cfg(not(feature = "private"))] -use serde::Serialize; -#[cfg(not(feature = "private"))] use uuid::Uuid; #[cfg(not(feature = "private"))] use windmill_common::s3_helpers::StorageResourceType; @@ -32,21 +30,6 @@ use axum::response::Response; #[cfg(all(feature = "parquet", not(feature = "private")))] use serde::Deserialize; -#[derive(Serialize)] -#[cfg(not(feature = "private"))] -pub struct UploadFileResponse { - pub file_key: String, -} - -#[derive(Deserialize)] -#[cfg(not(feature = "private"))] -pub struct LoadImagePreviewQuery { - #[allow(dead_code)] - pub file_key: String, - #[allow(dead_code)] - pub storage: Option, -} - #[derive(Deserialize)] #[cfg(not(feature = "private"))] pub struct DownloadFileQuery { @@ -167,3 +150,21 @@ pub struct DeleteS3FileQuery { pub file_key: String, pub storage: Option, } + +#[cfg(not(feature = "private"))] +pub async fn get_workspace_s3_resource_and_check_paths<'c>( + _authed: &crate::db::ApiAuthed, + _db: &crate::db::DB, + _user_db: Option, + _token: &str, + _w_id: &str, + _storage: Option, + _paths: &[(&str, windmill_common::s3_helpers::S3Permission)], +) -> windmill_common::error::Result<( + Option, + Option, +)> { + Err(windmill_common::error::Error::internal_err( + "Not implemented in Windmill's Open Source repository".to_string(), + )) +} diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 5071730b51..33beb32ec6 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -30,19 +30,22 @@ use windmill_common::auth::is_super_admin_email; use windmill_common::auth::TOKEN_PREFIX_LEN; use windmill_common::db::UserDbWithAuthed; use windmill_common::error::JsonResult; +use windmill_common::flow_conversations::add_message_to_conversation_tx; use windmill_common::flow_status::{JobResult, RestartedFrom}; use windmill_common::jobs::{ check_tag_available_for_workspace_internal, format_completed_job_result, format_result, - ENTRYPOINT_OVERRIDE, + DynamicInput, ENTRYPOINT_OVERRIDE, }; -use windmill_common::utils::WarnAfterExt; +use windmill_common::s3_helpers::{upload_artifact_to_store, BundleFormat}; +use windmill_common::utils::{RunnableKind, WarnAfterExt}; use windmill_common::worker::{Connection, CLOUD_HOSTED, TMP_DIR}; +use windmill_common::DYNAMIC_INPUT_CACHE; #[cfg(all(feature = "enterprise", feature = "smtp"))] use windmill_common::{email_oss::send_email_html, server::load_smtp_config}; -use windmill_common::scripts::PREVIEW_IS_CODEBASE_HASH; use windmill_common::variables::get_workspace_key; +use crate::triggers::trigger_helpers::ScriptId; use crate::{ add_webhook_allowed_origin, args::{self, RawWebhookArgs}, @@ -100,6 +103,9 @@ use windmill_queue::{ PushArgsOwned, PushIsolationLevel, }; +use crate::flow_conversations; +use windmill_common::flow_conversations::MessageType; + pub fn workspaced_service() -> Router { let cors = CorsLayer::new() .allow_methods([http::Method::GET, http::Method::POST]) @@ -170,6 +176,30 @@ pub fn workspaced_service() -> Router { .layer(cors.clone()) .layer(ce_headers.clone()), ) + .route( + "/run_and_stream/f/*script_path", + get(stream_flow_by_path) + .post(stream_flow_by_path) + .head(|| async { "" }) + .layer(cors.clone()) + .layer(ce_headers.clone()), + ) + .route( + "/run_and_stream/p/*script_path", + get(stream_script_by_path) + .post(stream_script_by_path) + .head(|| async { "" }) + .layer(cors.clone()) + .layer(ce_headers.clone()), + ) + .route( + "/run_and_stream/h/:hash", + get(stream_script_by_hash) + .post(stream_script_by_hash) + .head(|| async { "" }) + .layer(cors.clone()) + .layer(ce_headers.clone()), + ) .route( "/run/h/:hash", post(run_job_by_hash) @@ -192,6 +222,7 @@ pub fn workspaced_service() -> Router { "/run_wait_result/preview_flow", post(run_wait_result_preview_flow), ) + .route("/run/dynamic_select", post(run_dynamic_select)) .route("/list", get(list_jobs)) .route( "/list_selected_job_groups", @@ -288,6 +319,7 @@ pub fn workspace_unauthed_service() -> Router { get(get_completed_job_logs_tail), ) .route("/get_args/:id", get(get_args)) + .route("/queue/get_started_at_by_ids", post(get_started_at_by_ids)) .route("/get_flow_debug_info/:id", get(get_flow_job_debug_info)) .route("/completed/get/:id", get(get_completed_job)) .route("/completed/get_result/:id", get(get_completed_job_result)) @@ -746,7 +778,7 @@ macro_rules! get_job_query { ("v2_job_completed", $($opts:tt)*) => { get_job_query!( @impl "v2_job_completed", ($($opts)*), - "v2_job_completed.duration_ms, CASE WHEN status = 'success' OR status = 'skipped' THEN true ELSE false END as success, result_columns, deleted, status = 'skipped' as is_skipped, result->'wm_labels' as labels, \ + "v2_job_completed.duration_ms, v2_job_completed.completed_at, CASE WHEN status = 'success' OR status = 'skipped' THEN true ELSE false END as success, result_columns, deleted, status = 'skipped' as is_skipped, result->'wm_labels' as labels, \ CASE WHEN result is null or pg_column_size(result) < 90000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result", "", ) @@ -809,101 +841,6 @@ macro_rules! get_job_query { } } -// CREATE OR REPLACE VIEW v2_as_queue AS -// SELECT -// j.id, -// j.workspace_id, -// j.parent_job, -// j.created_by, -// j.created_at, -// q.started_at, -// q.scheduled_for, -// q.running, -// j.runnable_id AS script_hash, -// j.runnable_path AS script_path, -// j.args, -// j.raw_code, -// q.canceled_by IS NOT NULL AS canceled, -// q.canceled_by, -// q.canceled_reason, -// r.ping AS last_ping, -// j.kind AS job_kind, -// CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END -// AS schedule_path, -// j.permissioned_as, -// COALESCE(s.flow_status, s.workflow_as_code_status) AS flow_status, -// j.raw_flow, -// j.flow_step_id IS NOT NULL AS is_flow_step, -// j.script_lang AS language, -// q.suspend, -// q.suspend_until, -// j.same_worker, -// j.raw_lock, -// j.pre_run_error, -// j.permissioned_as_email AS email, -// j.visible_to_owner, -// r.memory_peak AS mem_peak, -// j.flow_innermost_root_job AS root_job, -// s.flow_leaf_jobs AS leaf_jobs, -// j.tag, -// j.concurrent_limit, -// j.concurrency_time_window_s, -// j.timeout, -// j.flow_step_id, -// j.cache_ttl, -// j.priority, -// NULL::TEXT AS logs, -// j.script_entrypoint_override, -// j.preprocessed -// FROM v2_job_queue q -// JOIN v2_job j USING (id) -// LEFT JOIN v2_job_runtime r USING (id) -// LEFT JOIN v2_job_status s USING (id) -// ; - -// -- Add up migration script here -// CREATE OR REPLACE VIEW v2_as_completed_job AS -// SELECT -// j.id, -// j.workspace_id, -// j.parent_job, -// j.created_by, -// j.created_at, -// c.duration_ms, -// c.status = 'success' OR c.status = 'skipped' AS success, -// j.runnable_id AS script_hash, -// j.runnable_path AS script_path, -// j.args, -// c.result, -// FALSE AS deleted, -// j.raw_code, -// c.status = 'canceled' AS canceled, -// c.canceled_by, -// c.canceled_reason, -// j.kind AS job_kind, -// CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END -// AS schedule_path, -// j.permissioned_as, -// COALESCE(c.flow_status, c.workflow_as_code_status) AS flow_status, -// j.raw_flow, -// j.flow_step_id IS NOT NULL AS is_flow_step, -// j.script_lang AS language, -// c.started_at, -// c.status = 'skipped' AS is_skipped, -// j.raw_lock, -// j.permissioned_as_email AS email, -// j.visible_to_owner, -// c.memory_peak AS mem_peak, -// j.tag, -// j.priority, -// NULL::TEXT AS logs, -// c.result_columns, -// j.script_entrypoint_override, -// j.preprocessed -// FROM v2_job_completed c -// JOIN v2_job j USING (id) -// ; - #[derive(Copy, Clone)] struct GetQuery<'a> { with_logs: bool, @@ -1474,10 +1411,10 @@ async fn get_job_logs( .flatten(); let record = sqlx::query!( - "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index - FROM v2_as_completed_job - LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id - WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))", + "SELECT j.created_by AS \"created_by!\", CONCAT(coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index + FROM v2_job j + LEFT JOIN job_logs ON job_logs.job_id = j.id + WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", id, w_id, tags.as_ref().map(|v| v.as_slice()) @@ -1522,10 +1459,10 @@ async fn get_job_logs( Ok(content_plain(Body::from(logs))) } else { let text = sqlx::query!( - "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index - FROM v2_as_queue - LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id - WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))", + "SELECT j.created_by AS \"created_by!\", CONCAT(coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index + FROM v2_job j + LEFT JOIN job_logs ON job_logs.job_id = j.id + WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", id, w_id, tags.as_ref().map(|v| v.as_slice()) @@ -1581,9 +1518,9 @@ async fn get_args( .map(|authed| get_scope_tags(authed)) .flatten(); let record = sqlx::query!( - "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\" - FROM v2_as_completed_job - WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "SELECT j.created_by AS \"created_by!\", j.args as \"args: sqlx::types::Json>\" + FROM v2_job j + WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", id, &w_id, tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, @@ -1639,6 +1576,32 @@ async fn get_args( } } +async fn get_started_at_by_ids( + Extension(db): Extension, + Json(mut ids): Json>, +) -> JsonResult>>> { + ids.truncate(100); + + let started_at = sqlx::query!( + "SELECT id, started_at FROM v2_job_queue WHERE id = ANY($1)", + ids.as_slice() + ) + .fetch_all(&db) + .await?; + + let as_map = started_at + .iter() + .map(|x| (x.id, x.started_at)) + .collect::>(); + + let mut r = Vec::new(); + for id in ids { + r.push(as_map.get(&id).map(|x| x.clone()).unwrap_or_default()); + } + + Ok(Json(r)) +} + #[derive(Debug, sqlx::FromRow, Serialize)] pub struct ListableCompletedJob { pub r#type: String, @@ -1700,6 +1663,8 @@ pub struct RunJobQuery { pub timeout: Option, pub cache_ttl: Option, pub skip_preprocessor: Option, + pub poll_delay_ms: Option, + pub memory_id: Option, } impl RunJobQuery { @@ -1760,8 +1725,8 @@ impl From for ListQueueQuery { created_by: lcq.created_by, started_before: lcq.started_before, started_after: lcq.started_after, - created_before: lcq.created_before, - created_after: lcq.created_after, + created_before: lcq.created_before_queue.or(lcq.created_before), + created_after: lcq.created_after_queue.or(lcq.created_after), created_or_started_before: lcq.created_or_started_before, created_or_started_after: lcq.created_or_started_after, worker: lcq.worker, @@ -1795,11 +1760,11 @@ pub fn filter_list_queue_query( if join_outstanding_wait_times { sqlb.left() .join("outstanding_wait_time") - .on_eq("v2_job.id", "outstanding_wait_time.job_id"); + .on_eq("v2_job_queue.id", "outstanding_wait_time.job_id"); } if w_id != "admins" || !lq.all_workspaces.is_some_and(|x| x) { - sqlb.and_where_eq("v2_job.workspace_id", "?".bind(&w_id)); + sqlb.and_where_eq("v2_job_queue.workspace_id", "?".bind(&w_id)); } if let Some(w) = &lq.worker { @@ -1900,8 +1865,7 @@ pub fn filter_list_queue_query( } if lq.is_not_schedule.unwrap_or(false) { - sqlb.and_where("trigger_kind != 'schedule'") - .or_where("trigger_kind IS NULL"); + sqlb.and_where("trigger_kind IS DISTINCT FROM 'schedule'"); } sqlb @@ -2004,6 +1968,7 @@ async fn cancel_jobs( db: &DB, username: &str, w_id: &str, + force_cancel: bool, ) -> error::JsonResult> { let mut uuids = vec![]; let mut tx = db.begin().await?; @@ -2063,7 +2028,7 @@ async fn cancel_jobs( w_id, tx, db, - false, + force_cancel, false, ) .await?; @@ -2094,17 +2059,23 @@ async fn cancel_jobs( Ok(Json(uuids)) } +#[derive(Deserialize)] +pub struct CancelSelectionQuery { + force_cancel: Option, +} + async fn cancel_selection( authed: ApiAuthed, Extension(db): Extension, Extension(user_db): Extension, Path(w_id): Path, + Query(query): Query, Json(jobs): Json>, ) -> error::JsonResult> { let mut tx = user_db.begin(&authed).await?; let tags = get_scope_tags(&authed).map(|v| v.iter().map(|s| s.to_string()).collect_vec()); let jobs_to_cancel = sqlx::query_scalar!( - "SELECT id AS \"id!\" FROM v2_as_queue WHERE id = ANY($1) AND schedule_path IS NULL AND ($2::text[] IS NULL OR tag = ANY($2))", + "SELECT j.id AS \"id!\" FROM v2_job j WHERE j.id = ANY($1) AND j.trigger_kind != 'schedule'::job_trigger_kind AND ($2::text[] IS NULL OR j.tag = ANY($2))", &jobs, tags.as_ref().map(|v| v.as_slice()) ) @@ -2112,7 +2083,14 @@ async fn cancel_selection( .await?; tx.commit().await?; - cancel_jobs(jobs_to_cancel, &db, authed.username.as_str(), w_id.as_str()).await + cancel_jobs( + jobs_to_cancel, + &db, + authed.username.as_str(), + w_id.as_str(), + query.force_cancel.unwrap_or(false), + ) + .await } async fn list_filtered_job_uuids( @@ -2201,7 +2179,7 @@ async fn count_queue_jobs( Ok(Json( sqlx::query_as!( QueueStats, - "SELECT coalesce(COUNT(*) FILTER(WHERE suspend = 0 AND running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE suspend > 0), 0) as \"suspended!\" FROM v2_as_queue WHERE (workspace_id = $1 OR $2) AND scheduled_for <= now() AND ($3::text[] IS NULL OR tag = ANY($3))", + "SELECT coalesce(COUNT(*) FILTER(WHERE q.suspend = 0 AND q.running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE q.suspend > 0), 0) as \"suspended!\" FROM v2_job_queue q JOIN v2_job j USING (id) WHERE (j.workspace_id = $1 OR $2) AND q.scheduled_for <= now() AND ($3::text[] IS NULL OR j.tag = ANY($3))", w_id, w_id == "admins" && cq.all_workspaces.unwrap_or(false), tags.as_ref().map(|v| v.as_slice()) @@ -2286,10 +2264,13 @@ async fn list_jobs( Query(pagination): Query, Query(lq): Query, ) -> error::JsonResult> { - let limit = pagination.per_page.unwrap_or(1000); let (per_page, offset) = paginate(pagination); let lqc = lq.clone(); + if offset > 0 { + tracing::warn!("offset is not 0, but is ignored for list_jobs. Use created_before or completed_before instead."); + } + if lq.success.is_some() && lq.running.is_some_and(|x| x) { return Err(error::Error::BadRequest( "cannot specify both success and running".to_string(), @@ -2298,7 +2279,7 @@ async fn list_jobs( let sqlc = if lq.running.is_none() { Some(list_completed_jobs_query( &w_id, - Some(per_page + offset), + Some(per_page), 0, &ListCompletedQuery { order_desc: Some(true), ..lqc }, UnifiedJob::completed_job_fields(), @@ -2311,26 +2292,22 @@ async fn list_jobs( let sql = if lq.success.is_none() && lq.label.is_none() - && lq.created_or_started_before.is_none() + && lq.created_before.is_none() && lq.started_before.is_none() + && lq.created_or_started_before.is_none() + && lq.completed_before.is_none() { let mut sqlq = list_queue_jobs_query( &w_id, &ListQueueQuery { order_desc: Some(true), ..lq.into() }, UnifiedJob::queued_job_fields(), - Pagination { per_page: Some(limit), page: None }, + Pagination { per_page: None, page: None }, true, get_scope_tags(&authed), ); if let Some(sqlc) = sqlc { - format!( - "{} UNION ALL {} LIMIT {} OFFSET {};", - &sqlq.subquery()?, - &sqlc.subquery()?, - per_page, - offset - ) + format!("{} UNION ALL {}", &sqlq.subquery()?, &sqlc.subquery()?,) } else { sqlq.limit(per_page).offset(offset).query()? } @@ -2344,6 +2321,7 @@ async fn list_jobs( } sqlc.unwrap().limit(per_page).offset(offset).query()? }; + // tracing::info!("sql: {}", sql); let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; let jobs: Vec = sqlx::query_as(&sql) @@ -2644,9 +2622,9 @@ async fn get_suspended_flow_info<'c>( let flow = sqlx::query_as!( FlowInfo, r#" - SELECT id AS "id!", flow_status, suspend AS "suspend!", script_path - FROM v2_as_queue - WHERE id = $1 + SELECT j.id AS "id!", COALESCE(s.flow_status, s.workflow_as_code_status) as flow_status, q.suspend AS "suspend!", j.runnable_path as script_path + FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_status s USING (id) + WHERE j.id = $1 "#, job_id, ) @@ -2860,9 +2838,9 @@ pub async fn get_flow_user_state( let mut tx = user_db.begin(&authed).await?; let r = sqlx::query_scalar!( r#" - SELECT flow_status->'user_states'->$1 - FROM v2_as_queue - WHERE id = $2 AND workspace_id = $3 + SELECT COALESCE(s.flow_status, s.workflow_as_code_status)->'user_states'->$1 + FROM v2_job_queue q LEFT JOIN v2_job_status s USING (id) + WHERE q.id = $2 AND q.workspace_id = $3 "#, key, job_id, @@ -3221,6 +3199,7 @@ pub struct UnifiedJob { pub created_by: String, pub created_at: chrono::DateTime, pub started_at: Option>, + pub completed_at: Option>, pub scheduled_for: Option>, pub running: Option, pub script_hash: Option, @@ -3253,13 +3232,14 @@ pub struct UnifiedJob { const CJ_FIELDS: &[&str] = &[ "'CompletedJob' as typ", - "v2_job.id", - "v2_job.workspace_id", + "v2_job_completed.id", + "v2_job_completed.workspace_id", "v2_job.parent_job", "v2_job.created_by", "v2_job.created_at", "v2_job_completed.started_at", "null as scheduled_for", + "v2_job_completed.completed_at", "null as running", "v2_job.runnable_id as script_hash", "v2_job.runnable_path as script_path", @@ -3292,13 +3272,14 @@ const CJ_FIELDS: &[&str] = &[ const QJ_FIELDS: &[&str] = &[ "'QueuedJob' as typ", - "v2_job.id", - "v2_job.workspace_id", + "v2_job_queue.id", + "v2_job_queue.workspace_id", "v2_job.parent_job", "v2_job.created_by", - "v2_job.created_at", + "v2_job_queue.created_at", "v2_job_queue.started_at", "v2_job_queue.scheduled_for", + "null as completed_at", "v2_job_queue.running", "v2_job.runnable_id as script_hash", "v2_job.runnable_path as script_path", @@ -3351,6 +3332,7 @@ impl<'a> From for Job { created_by: uj.created_by, created_at: uj.created_at, started_at: uj.started_at, + completed_at: uj.completed_at, duration_ms: uj.duration_ms.unwrap(), success: uj.success.unwrap(), script_hash: uj.script_hash, @@ -3457,6 +3439,7 @@ struct Preview { tag: Option, dedicated_worker: Option, lock: Option, + format: Option, } #[derive(Deserialize)] @@ -3473,6 +3456,22 @@ struct PreviewFlow { restarted_from: Option, } +#[derive(Debug, Deserialize)] +struct DynamicSelectRequest { + pub entrypoint_function: String, + pub args: Option>>, + pub runnable_ref: DynamicSelectRunnableRef, +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "source")] +pub enum DynamicSelectRunnableRef { + #[serde(rename = "deployed")] + Deployed { path: String, runnable_kind: RunnableKind }, + #[serde(rename = "inline")] + Inline { code: String, lang: Option }, +} + pub struct QueryOrBody(pub Option); #[axum::async_trait] @@ -3514,7 +3513,7 @@ where } fn decode_payload(t: String) -> anyhow::Result { - let vec = base64::engine::general_purpose::URL_SAFE + let vec = base64::engine::general_purpose::STANDARD .decode(t) .context("invalid base64")?; serde_json::from_slice(vec.as_slice()).context("invalid json") @@ -3782,7 +3781,7 @@ async fn batch_rerun_handle_job( user_db.clone(), w_id.clone(), StripPath(job.script_path.clone()), - RunJobQuery { ..Default::default() }, + RunJobQuery { skip_preprocessor: Some(true), ..Default::default() }, PushArgsOwned { extra: None, args }, ) .await; @@ -3798,7 +3797,7 @@ async fn batch_rerun_handle_job( user_db.clone(), w_id.clone(), StripPath(job.script_path.clone()), - RunJobQuery { ..Default::default() }, + RunJobQuery { skip_preprocessor: Some(true), ..Default::default() }, PushArgsOwned { extra: None, args }, ) .await @@ -3809,7 +3808,7 @@ async fn batch_rerun_handle_job( user_db.clone(), w_id.clone(), job.script_hash, - RunJobQuery { ..Default::default() }, + RunJobQuery { skip_preprocessor: Some(true), ..Default::default() }, PushArgsOwned { extra: None, args }, ) .await @@ -3825,6 +3824,82 @@ async fn batch_rerun_handle_job( )) } +/// Set the memory_id in flow_status for agent memory persistence +async fn set_flow_memory_id( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + job_id: Uuid, + memory_id: Uuid, +) -> error::Result<()> { + sqlx::query!( + "UPDATE v2_job_status + SET flow_status = jsonb_set( + flow_status, + '{memory_id}', + to_jsonb($2::uuid) + ) + WHERE id = $1", + job_id, + memory_id + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +async fn handle_chat_conversation_messages( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + authed: &ApiAuthed, + w_id: &str, + flow_path: &str, + run_query: &RunJobQuery, + user_message_raw: Option<&Box>, +) -> error::Result<()> { + let memory_id = run_query.memory_id.ok_or_else(|| { + windmill_common::error::Error::BadRequest( + "memory_id is required for chat-enabled flows".to_string(), + ) + })?; + + let user_message_raw = user_message_raw.ok_or_else(|| { + windmill_common::error::Error::BadRequest( + "user_message argument is required for chat-enabled flows".to_string(), + ) + })?; + + // Deserialize the RawValue to get the actual string without quotes + let user_message: String = serde_json::from_str(user_message_raw.get()).map_err(|e| { + windmill_common::error::Error::BadRequest(format!( + "Failed to deserialize user_message: {}", + e + )) + })?; + + // Create conversation with provided ID (or get existing one) + flow_conversations::get_or_create_conversation_with_id( + tx, + w_id, + flow_path, + &authed.username, + &user_message, + memory_id, + ) + .await?; + + // Create user message + add_message_to_conversation_tx( + tx, + memory_id, + None, + &user_message, + MessageType::User, + None, + true, + ) + .await?; + + Ok(()) +} + pub async fn run_flow_by_path( authed: ApiAuthed, Extension(db): Extension, @@ -3871,6 +3946,7 @@ pub async fn run_flow_by_path_inner( tag, dedicated_worker, has_preprocessor, + chat_input_enabled, on_behalf_of_email, edited_by, early_return, @@ -3896,11 +3972,11 @@ pub async fn run_flow_by_path_inner( &authed.email, username_to_permissioned_as(&authed.username), Some(authed.clone().into()), - PushIsolationLevel::Isolated(user_db, authed.clone().into()), + PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()), ) }; - let (uuid, tx) = push( + let (uuid, mut tx) = push( &db, tx, &w_id, @@ -3932,8 +4008,29 @@ pub async fn run_flow_by_path_inner( None, push_authed.as_ref(), false, + None, + None, ) .await?; + + // Set memory_id if provided (for agent memory) + if let Some(memory_id) = run_query.memory_id { + set_flow_memory_id(&mut tx, uuid, memory_id).await?; + } + + // Handle conversation messages for chat-enabled flows + if chat_input_enabled.unwrap_or(false) { + handle_chat_conversation_messages( + &mut tx, + &authed, + &w_id, + &flow_path.to_string(), + &run_query, + args.args.get("user_message"), + ) + .await?; + } + tx.commit().await?; Ok((uuid, early_return)) } @@ -3974,10 +4071,10 @@ pub async fn restart_flow( let mut tx = user_db.clone().begin(&authed).await?; let completed_job = sqlx::query!( "SELECT - script_path, args AS \"args: sqlx::types::Json>>\", - tag AS \"tag!\", priority - FROM v2_as_completed_job - WHERE id = $1 and workspace_id = $2", + j.runnable_path as script_path, j.args AS \"args: sqlx::types::Json>>\", + j.tag AS \"tag!\", j.priority + FROM v2_job j + WHERE j.id = $1 and j.workspace_id = $2", job_id, &w_id, ) @@ -4028,6 +4125,8 @@ pub async fn restart_flow( completed_job.priority, Some(&authed.clone().into()), false, + None, + None, ) .await?; tx.commit().await?; @@ -4130,6 +4229,8 @@ pub async fn run_script_by_path_inner( None, push_authed.as_ref(), false, + None, + None, ) .await?; tx.commit().await?; @@ -4195,6 +4296,9 @@ pub async fn run_workflow_as_code( concurrency_time_window_s: job.concurrency_time_window_s, cache_ttl: job.cache_ttl, dedicated_worker: None, + // TODO(debouncing): enable for this mode + custom_debounce_key: None, + debounce_delay_s: None, }), Some(job.tag.clone()), None, @@ -4282,6 +4386,8 @@ pub async fn run_workflow_as_code( None, push_authed.as_ref(), false, + None, + None, ) .await?; @@ -4441,12 +4547,17 @@ pub async fn run_wait_result_internal( if result.is_none() { let row = sqlx::query!( - "SELECT + " + SELECT result AS \"result: sqlx::types::Json>\", result_columns, status = 'success' AS \"success!\" - FROM v2_job_completed - WHERE id = $1 AND workspace_id = $2", + FROM + v2_job_completed + WHERE + id = $1 AND + workspace_id = $2 + ", uuid, &w_id ) @@ -4613,10 +4724,9 @@ pub async fn delete_job_metadata_after_use(db: &DB, job_uuid: Uuid) -> Result<() pub async fn check_queue_too_long(db: &DB, queue_limit: Option) -> error::Result<()> { if let Some(limit) = queue_limit { let count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM v2_as_queue WHERE canceled = false AND (scheduled_for <= now() - OR (suspend_until IS NOT NULL - AND ( suspend <= 0 - OR suspend_until <= now())))", + "SELECT COUNT(*) FROM v2_job_queue q WHERE q.canceled_by IS NULL AND (q.scheduled_for <= now() + OR (q.suspend_until IS NOT NULL + AND (q.suspend <= 0 OR q.suspend_until <= now())))", ) .fetch_one(db) .await? @@ -4820,6 +4930,8 @@ pub async fn run_wait_result_job_by_path_get( None, push_authed.as_ref(), false, + None, + None, ) .await?; tx.commit().await?; @@ -4972,6 +5084,8 @@ pub async fn run_wait_result_script_by_path_internal( None, push_authed.as_ref(), false, + None, + None, ) .await?; tx.commit().await?; @@ -5014,6 +5128,8 @@ pub async fn run_wait_result_script_by_hash( concurrency_key, concurrent_limit, concurrency_time_window_s, + debounce_key, + debounce_delay_s, mut cache_ttl, language, dedicated_worker, @@ -5060,6 +5176,8 @@ pub async fn run_wait_result_script_by_hash( custom_concurrency_key: concurrency_key, concurrent_limit: concurrent_limit, concurrency_time_window_s: concurrency_time_window_s, + custom_debounce_key: debounce_key, + debounce_delay_s, cache_ttl, language, dedicated_worker, @@ -5088,6 +5206,8 @@ pub async fn run_wait_result_script_by_hash( None, push_authed.as_ref(), false, + None, + None, ) .await?; tx.commit().await?; @@ -5127,6 +5247,197 @@ pub async fn run_wait_result_flow_by_path( .await } +pub async fn stream_flow_by_path( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, flow_path)): Path<(String, StripPath)>, + Query(run_query): Query, + method: hyper::http::Method, + args: RawWebhookArgs, +) -> error::Result { + stream_job( + authed, + db, + user_db, + w_id, + RunnableId::from_flow_path(flow_path.to_path()), + args, + run_query, + method == http::Method::GET, + ) + .await +} + +pub async fn stream_script_by_path( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, script_path)): Path<(String, StripPath)>, + Query(run_query): Query, + method: hyper::http::Method, + args: RawWebhookArgs, +) -> error::Result { + stream_job( + authed, + db, + user_db, + w_id, + RunnableId::from_script_path(script_path.to_path()), + args, + run_query, + method == http::Method::GET, + ) + .await +} + +pub async fn stream_script_by_hash( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, script_hash)): Path<(String, ScriptHash)>, + Query(run_query): Query, + method: hyper::http::Method, + args: RawWebhookArgs, +) -> error::Result { + stream_job( + authed, + db, + user_db, + w_id, + RunnableId::from_script_hash(script_hash), + args, + run_query, + method == http::Method::GET, + ) + .await +} + +pub async fn stream_job( + authed: ApiAuthed, + db: DB, + user_db: UserDB, + w_id: String, + runnable_id: RunnableId, + args: RawWebhookArgs, + run_query: RunJobQuery, + is_get: bool, +) -> error::Result { + let args = if is_get { + let payload_r = run_query.payload.clone().map(decode_payload).map(|x| { + x.map_err(|e| { + Error::internal_err(format!("Impossible to decode query payload: {e:#?}")) + }) + }); + + let payload_args = if let Some(payload) = payload_r { + payload? + } else { + HashMap::new() + }; + + let mut args = args.process_args(&authed, &db, &w_id, None).await?; + args.body = args::Body::HashMap(payload_args); + + let args = args + .to_args_from_runnable(&db, &w_id, runnable_id.clone(), run_query.skip_preprocessor) + .await?; + args + } else { + args.to_args_from_runnable( + &authed, + &db, + &w_id, + runnable_id.clone(), + run_query.skip_preprocessor, + ) + .await? + }; + + let poll_delay_ms = run_query.poll_delay_ms; + let uuid = match runnable_id { + RunnableId::ScriptId(ScriptId::ScriptPath(script_path)) + | RunnableId::HubScript(script_path) => { + run_script_by_path_inner( + authed.clone(), + db.clone(), + user_db, + w_id.clone(), + StripPath(script_path), + run_query, + args, + ) + .await? + .0 + } + RunnableId::ScriptId(ScriptId::ScriptHash(script_hash)) => { + run_job_by_hash_inner( + authed.clone(), + db.clone(), + user_db, + w_id.clone(), + script_hash, + run_query, + args, + ) + .await? + .0 + } + RunnableId::FlowPath(flow_path) => { + run_flow_by_path_inner( + authed.clone(), + db.clone(), + user_db, + w_id.clone(), + StripPath(flow_path), + run_query, + args, + ) + .await? + .0 + } + }; + + let opt_authed = Some(authed.clone()); + let opt_tokened = OptTokened { token: None }; // ignored when authed is some + let (tx, rx) = tokio::sync::mpsc::channel(32); + + let stream = tokio_stream::wrappers::ReceiverStream::new(rx).map(|x| { + format!( + "data: {}\n\n", + serde_json::to_string(&x).unwrap_or_default() + ) + }); + + start_job_update_sse_stream( + opt_authed, + opt_tokened, + db, + w_id, + uuid, + None, + None, + None, + None, + Some(true), + Some(true), + None, + None, + tx, + poll_delay_ms, + ); + + let body = axum::body::Body::from_stream(stream.map(Result::<_, std::convert::Infallible>::Ok)); + + Ok(Response::builder() + .status(200) + .header("Content-Type", "text/event-stream") + .header("Cache-Control", "no-cache") + .header("Connection", "keep-alive") + .body(body) + .unwrap()) +} + pub async fn run_wait_result_flow_by_path_internal( db: sqlx::Pool, run_query: RunJobQuery, @@ -5149,6 +5460,7 @@ pub async fn run_wait_result_flow_by_path_internal( dedicated_worker, early_return, has_preprocessor, + chat_input_enabled, on_behalf_of_email, edited_by, version, @@ -5171,11 +5483,11 @@ pub async fn run_wait_result_flow_by_path_internal( &authed.email, username_to_permissioned_as(&authed.username), Some(authed.clone().into()), - PushIsolationLevel::Isolated(user_db, authed.clone().into()), + PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()), ) }; - let (uuid, tx) = push( + let (uuid, mut tx) = push( &db, tx, &w_id, @@ -5207,8 +5519,29 @@ pub async fn run_wait_result_flow_by_path_internal( None, push_authed.as_ref(), false, + None, + None, ) .await?; + + // Set conversation_id if provided (for agent memory) + if let Some(memory_id) = run_query.memory_id { + set_flow_memory_id(&mut tx, uuid, memory_id).await?; + } + + // Handle conversation messages for chat-enabled flows + if chat_input_enabled.unwrap_or(false) { + handle_chat_conversation_messages( + &mut tx, + &authed, + &w_id, + &flow_path.to_string(), + &run_query, + args.args.get("user_message"), + ) + .await?; + } + tx.commit().await?; run_wait_result(&db, uuid, w_id, early_return, &authed.username).await @@ -5253,6 +5586,8 @@ async fn run_preview_script( custom_concurrency_key: None, concurrent_limit: None, // TODO(gbouv): once I find out how to store limits in the content of a script, should be easy to plug limits here concurrency_time_window_s: None, // TODO(gbouv): same as above + custom_debounce_key: None, // TODO(pyra): same as for concurrency limits. + debounce_delay_s: None, cache_ttl: None, dedicated_worker: preview.dedicated_worker, }), @@ -5278,6 +5613,8 @@ async fn run_preview_script( None, Some(&authed.clone().into()), false, + None, + None, ) .await?; tx.commit().await?; @@ -5317,8 +5654,6 @@ async fn run_bundle_preview_script( Query(run_query): Query, mut multipart: axum::extract::Multipart, ) -> error::Result<(StatusCode, String)> { - use windmill_common::scripts::PREVIEW_IS_TAR_CODEBASE_HASH; - if authed.is_operator { return Err(error::Error::NotAuthorized( "Operators cannot run preview jobs for security reasons".to_string(), @@ -5329,6 +5664,7 @@ async fn run_bundle_preview_script( let mut tx = None; let mut uploaded = false; let mut is_tar = false; + let mut format = BundleFormat::Cjs; while let Some(field) = multipart.next_field().await.unwrap() { let name = field.name().unwrap().to_string(); @@ -5336,6 +5672,10 @@ async fn run_bundle_preview_script( let data = data.map_err(to_anyhow)?; if name == "preview" { let preview: Preview = serde_json::from_slice(&data).map_err(to_anyhow)?; + format = preview + .format + .and_then(|s| BundleFormat::from_string(&s)) + .unwrap_or(BundleFormat::Cjs); let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(preview.tag.clone()); @@ -5356,11 +5696,10 @@ async fn run_bundle_preview_script( ltx, &w_id, JobPayload::Code(RawCode { - hash: if is_tar { - Some(PREVIEW_IS_TAR_CODEBASE_HASH) - } else { - Some(PREVIEW_IS_CODEBASE_HASH) - }, + hash: Some(windmill_common::scripts::codebase_to_hash( + is_tar, + format == BundleFormat::Esm, + )), content: preview.content.unwrap_or_default(), path: preview.path, language: preview.language.unwrap_or(ScriptLang::Deno), @@ -5370,6 +5709,8 @@ async fn run_bundle_preview_script( cache_ttl: None, dedicated_worker: preview.dedicated_worker, custom_concurrency_key: None, + custom_debounce_key: None, + debounce_delay_s: None, }), PushArgs::from(&args), authed.display_username(), @@ -5392,6 +5733,8 @@ async fn run_bundle_preview_script( None, Some(&authed.clone().into()), false, + None, + None, ) .await?; job_id = Some(uuid); @@ -5409,52 +5752,22 @@ async fn run_bundle_preview_script( // tracing::info!("is_tar 2: {is_tar}"); + if format == BundleFormat::Esm { + id = format!("{}.esm", id); + } if is_tar { id = format!("{}.tar", id); } uploaded = true; - #[cfg(all(feature = "enterprise", feature = "parquet"))] - let object_store = windmill_common::s3_helpers::get_object_store().await; - - #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - let object_store: Option<()> = None; - - if &windmill_common::utils::MODE_AND_ADDONS.mode - == &windmill_common::utils::Mode::Standalone - && object_store.is_none() - { - std::fs::create_dir_all( - windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR.clone(), - )?; - windmill_common::worker::write_file_bytes( - &windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR, - &id, - &data, - )?; - } else { - #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - { - return Err(Error::ExecutionErr("codebase is an EE feature".to_string())); - } - - #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = object_store { - check_license_key_valid().await?; - - let path = windmill_common::s3_helpers::bundle(&w_id, &id); - if let Err(e) = os - .put(&object_store::path::Path::from(path.clone()), data.into()) - .await - { - tracing::info!("Failed to put snapshot to s3 at {path}: {:?}", e); - return Err(Error::ExecutionErr(format!("Failed to put {path} to s3"))); - } - } else { - return Err(Error::BadConfig("Object store is required for snapshot script and is not configured for servers".to_string())); - } - } + let path = windmill_common::s3_helpers::bundle(&w_id, &id); + upload_artifact_to_store( + &path, + data, + &windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR, + ) + .await?; } // println!("Length of `{}` is {} bytes", name, data.len()); } @@ -5559,6 +5872,8 @@ async fn run_dependencies_job( None, Some(&authed.clone().into()), false, + None, + None, ) .await?; tx.commit().await?; @@ -5626,6 +5941,8 @@ async fn run_flow_dependencies_job( None, Some(&authed.clone().into()), false, + None, + None, ) .await?; tx.commit().await?; @@ -5969,6 +6286,8 @@ async fn run_preview_flow_job( None, Some(&authed.clone().into()), false, + None, + None, ) .await?; tx.commit().await?; @@ -6000,6 +6319,159 @@ async fn run_wait_result_preview_flow( return result; } +async fn run_dynamic_select( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Query(run_query): Query, + Json(request): Json, +) -> error::Result { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + + if matches!( + request.runnable_ref, + DynamicSelectRunnableRef::Inline { .. } + ) && authed.is_operator + { + return Err(error::Error::NotAuthorized( + "Operators cannot run preview jobs for security reasons".to_string(), + )); + } + + let dynamic_input: DynamicInput; + + match request.runnable_ref { + DynamicSelectRunnableRef::Deployed { path, runnable_kind } => match runnable_kind { + RunnableKind::Script => { + let mut script_args = request.args.unwrap_or_default(); + script_args.insert( + "_ENTRYPOINT_OVERRIDE".to_string(), + serde_json::value::to_raw_value(&request.entrypoint_function)?, + ); + + let push_args = PushArgsOwned { extra: None, args: script_args.clone() }; + + let (uuid, _) = run_script_by_path_inner( + authed.clone(), + db.clone(), + user_db.clone(), + w_id.clone(), + StripPath(path), + run_query.clone(), + push_args.clone(), + ) + .await?; + + return Ok((StatusCode::CREATED, uuid.to_string()).into_response()); + } + RunnableKind::Flow => { + let mut conn = user_db.clone().begin(&authed).await?; + + let dynamic_input_res = match DYNAMIC_INPUT_CACHE.get(&format!("{}:{}", w_id, path)) + { + Some(cached) => cached.as_ref().clone(), + None => { + let dynamic_input = sqlx::query_scalar!( + r#" + SELECT + schema + FROM + flow + WHERE + workspace_id = $1 AND + path = $2 + "#, + &w_id, + &path + ) + .fetch_one(&mut *conn) + .await? + .and_then(|dynamic_input| { + Some(serde_json::from_value::(dynamic_input)) + }) + .transpose()?; + + let Some(dynamic_input) = dynamic_input else { + return Err(Error::BadRequest(format!( + "Flow at path {} does not have a dynamic select schema", + path + ))); + }; + + let dynamic_input_key = + windmill_common::jobs::generate_dynamic_input_key(&w_id, &path); + DYNAMIC_INPUT_CACHE + .insert(dynamic_input_key, Arc::new(dynamic_input.clone())); + dynamic_input + } + }; + + conn.commit().await?; + + dynamic_input = dynamic_input_res; + } + }, + DynamicSelectRunnableRef::Inline { code, lang: language } => { + dynamic_input = DynamicInput { + x_windmill_dyn_select_code: code, + x_windmill_dyn_select_lang: language.unwrap_or_default(), + }; + } + } + + let scheduled_for = run_query.get_scheduled_for(&db).await?; + let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); + + let (uuid, tx) = push( + &db, + tx, + &w_id, + JobPayload::Code(RawCode { + hash: None, + content: dynamic_input.x_windmill_dyn_select_code, + path: None, + language: dynamic_input.x_windmill_dyn_select_lang, + lock: None, + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + custom_debounce_key: None, + debounce_delay_s: None, + }), + PushArgs::from(&request.args.unwrap_or_default()), + authed.display_username(), + &authed.email, + username_to_permissioned_as(&authed.username), + authed.token_prefix.as_deref(), + scheduled_for, + None, + None, + None, + None, + run_query.job_id, + false, + false, + None, + true, + None, + run_query.timeout, + None, + None, + Some(&authed.clone().into()), + false, + None, + None, + ) + .await?; + tx.commit().await?; + + Ok((StatusCode::CREATED, uuid.to_string()).into_response()) +} + pub async fn run_job_by_hash( authed: ApiAuthed, Extension(db): Extension, @@ -6044,6 +6516,8 @@ pub async fn run_job_by_hash_inner( concurrency_key, concurrent_limit, concurrency_time_window_s, + debounce_delay_s, + debounce_key, mut cache_ttl, language, dedicated_worker, @@ -6092,6 +6566,8 @@ pub async fn run_job_by_hash_inner( custom_concurrency_key: concurrency_key, concurrent_limit: concurrent_limit, concurrency_time_window_s: concurrency_time_window_s, + custom_debounce_key: debounce_key, + debounce_delay_s, cache_ttl, language, dedicated_worker, @@ -6120,6 +6596,8 @@ pub async fn run_job_by_hash_inner( None, push_authed.as_ref(), false, + None, + None, ) .await?; tx.commit().await?; @@ -6136,6 +6614,8 @@ pub struct JobUpdateQuery { pub no_logs: Option, pub only_result: Option, pub fast: Option, + pub is_flow: Option, + pub poll_delay_ms: Option, } #[derive(Serialize, Debug)] @@ -6164,6 +6644,8 @@ pub struct JobUpdate { pub job: Option, #[serde(skip_serializing_if = "Option::is_none")] pub only_result: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_stream_job_id: Option, } impl JobUpdate { @@ -6182,6 +6664,7 @@ impl Hash for JobUpdate { self.mem_peak.hash(state); self.progress.hash(state); self.stream_offset.hash(state); + self.flow_stream_job_id.hash(state); if !self.completed.unwrap_or(false) { self.flow_status.as_ref().map(|x| x.get().hash(state)); self.workflow_as_code_status @@ -6254,6 +6737,7 @@ async fn get_job_update( running, only_result, no_logs, + is_flow, .. }): Query, ) -> JsonResult { @@ -6266,12 +6750,14 @@ async fn get_job_update( &job_id, log_offset, stream_offset, - get_progress, + get_progress.unwrap_or(false), running, true, false, only_result, no_logs, + is_flow, + None, ) .await?, )) @@ -6290,9 +6776,13 @@ async fn get_job_update_sse( no_logs, only_result, fast, + is_flow, + poll_delay_ms, }): Query, -) -> Response { - let stream = get_job_update_sse_stream( +) -> error::Result { + let (tx, rx) = tokio::sync::mpsc::channel(32); + + start_job_update_sse_stream( opt_authed, opt_tokened, db, @@ -6305,8 +6795,12 @@ async fn get_job_update_sse( only_result, fast, no_logs, - ) - .map(|x| { + is_flow, + tx, + poll_delay_ms, + ); + + let stream = tokio_stream::wrappers::ReceiverStream::new(rx).map(|x| { format!( "data: {}\n\n", serde_json::to_string(&x).unwrap_or_default() @@ -6315,26 +6809,31 @@ async fn get_job_update_sse( let body = axum::body::Body::from_stream(stream.map(Result::<_, std::convert::Infallible>::Ok)); - Response::builder() + Ok(Response::builder() .status(200) .header("Content-Type", "text/event-stream") .header("Cache-Control", "no-cache") .header("Connection", "keep-alive") .body(body) - .unwrap() + .unwrap()) } #[derive(Serialize)] #[serde(tag = "type", rename_all = "lowercase")] -enum JobUpdateSSEStream { +pub enum JobUpdateSSEStream { Update(JobUpdate), - Error(String), + Error { error: String }, NotFound, Timeout, Ping, } -fn get_job_update_sse_stream( +lazy_static::lazy_static! { + pub static ref TIMEOUT_SSE_STREAM: u64 = + std::env::var("TIMEOUT_SSE_STREAM").unwrap_or("60".to_string()).parse::().unwrap_or(60); +} + +pub fn start_job_update_sse_stream( opt_authed: Option, opt_tokened: OptTokened, db: DB, @@ -6347,13 +6846,15 @@ fn get_job_update_sse_stream( only_result: Option, fast: Option, no_logs: Option, -) -> impl futures::Stream { - let (tx, rx) = tokio::sync::mpsc::channel(32); - + is_flow: Option, + tx: tokio::sync::mpsc::Sender, + poll_delay_ms: Option, +) -> () { tokio::spawn(async move { let mut log_offset = initial_log_offset; let mut stream_offset = initial_stream_offset; let mut last_update_hash: Option = None; + let mut flow_stream_job_id = None; // Send initial update immediately let mut running = running; @@ -6367,12 +6868,14 @@ fn get_job_update_sse_stream( &job_id, log_offset, stream_offset, - get_progress, + false, running, true, true, only_result, no_logs, + is_flow, + flow_stream_job_id, ) .await { @@ -6399,6 +6902,9 @@ fn get_job_update_sse_stream( update.stream_offset = None; } } + if update.flow_stream_job_id.is_some() { + flow_stream_job_id = update.flow_stream_job_id; + } if tx.send(JobUpdateSSEStream::Update(update)).await.is_err() { tracing::warn!("Failed to send initial job update for job {job_id}"); return; @@ -6409,7 +6915,7 @@ fn get_job_update_sse_stream( } Err(e) => { if tx - .send(JobUpdateSSEStream::Error(e.to_string())) + .send(JobUpdateSSEStream::Error { error: e.to_string() }) .await .is_err() { @@ -6419,20 +6925,38 @@ fn get_job_update_sse_stream( } } + let mut get_progress_m: bool = false; // Poll for updates every 1 second let mut i = 0; let start = Instant::now(); let mut last_ping = Instant::now(); - + let mut last_progress_check = Instant::now(); loop { i += 1; - let ms_duration = if i > 100 || !fast.unwrap_or(false) { + + #[allow(unused_mut)] + let mut ms_duration = if i > 100 || !fast.unwrap_or(false) { 3000 } else if i > 10 { 500 } else { 100 }; + + #[allow(unused_variables)] + if let Some(poll_delay_ms) = poll_delay_ms { + #[cfg(feature = "enterprise")] + if poll_delay_ms < 50 { + tracing::warn!("Poll delay ms is less than 50, setting it to 50"); + ms_duration = 50; + } else { + ms_duration = poll_delay_ms; + } + + #[cfg(not(feature = "enterprise"))] + tracing::warn!("Settable poll delay requires EE"); + } + if last_ping.elapsed().as_secs() > 5 { if tx.send(JobUpdateSSEStream::Ping).await.is_err() { tracing::warn!("Failed to send job ping for job {job_id}"); @@ -6441,7 +6965,7 @@ fn get_job_update_sse_stream( last_ping = Instant::now(); } - if start.elapsed().as_secs() > 30 { + if start.elapsed().as_secs() > *TIMEOUT_SSE_STREAM { if tx.send(JobUpdateSSEStream::Timeout).await.is_err() { tracing::warn!("Failed to send job timeout for job {job_id}"); } @@ -6449,6 +6973,10 @@ fn get_job_update_sse_stream( } tokio::time::sleep(std::time::Duration::from_millis(ms_duration)).await; + // Check progress if the user requested it, and check periodically if the job has progress + // Once it has progress, we always check progress + let check_progress = get_progress.unwrap_or(false) + && (get_progress_m || last_progress_check.elapsed().as_secs() > 5); match get_job_update_data( &opt_authed, &opt_tokened, @@ -6457,12 +6985,14 @@ fn get_job_update_sse_stream( &job_id, log_offset, stream_offset, - get_progress, + check_progress, running, false, true, only_result, no_logs, + is_flow, + flow_stream_job_id, ) .await { @@ -6476,6 +7006,13 @@ fn get_job_update_sse_stream( if update.new_logs.as_ref().is_some_and(|x| x.is_empty()) { update.new_logs = None; } + if check_progress { + if update.progress.is_some() { + get_progress_m = true; + } else { + last_progress_check = Instant::now(); + } + } // if !only_result.unwrap_or(false) { // tracing::error!("update {:?}", update); @@ -6486,11 +7023,17 @@ fn get_job_update_sse_stream( // Update log offset if available if let Some(new_offset) = update.log_offset { if new_offset != log_offset.unwrap_or(0) { + // let logs = update.new_logs.clone().unwrap_or_default(); + // tracing::error!( + // "new_offset: {new_offset}; log_offset: {log_offset:?}; {} {logs}", + // logs.len() + // ); log_offset = Some(new_offset); } else { update.log_offset = None; } } + if let Some(new_stream_offset) = update.stream_offset { if new_stream_offset != stream_offset.unwrap_or(0) { stream_offset = Some(new_stream_offset); @@ -6498,6 +7041,13 @@ fn get_job_update_sse_stream( update.stream_offset = None; } } + if update.flow_stream_job_id.is_some() { + if flow_stream_job_id.is_none() { + flow_stream_job_id = update.flow_stream_job_id; + } else { + update.flow_stream_job_id = None; + } + } if let Some(new_mem_peak) = update.mem_peak { if new_mem_peak != mem_peak { mem_peak = new_mem_peak; @@ -6526,8 +7076,35 @@ fn get_job_update_sse_stream( } } }); +} - tokio_stream::wrappers::ReceiverStream::new(rx) +async fn get_flow_stream_delta( + db: &DB, + flow_stream_job_id: Option, + stream_offset: Option, +) -> error::Result, Option)>> { + if let Some(job_id) = flow_stream_job_id { + let record = sqlx::query!( + " + SELECT + string_agg(stream, '' order by idx asc) as stream, + max(idx) + 1 as offset + FROM job_result_stream_v2 + WHERE job_id = $2 AND idx >= $1 + ", + stream_offset.unwrap_or(0), + job_id, + ) + .fetch_optional(db) + .await?; + if let Some(record) = record { + Ok(Some((record.stream, record.offset))) + } else { + Ok(None) + } + } else { + Ok(None) + } } async fn get_job_update_data( @@ -6538,12 +7115,14 @@ async fn get_job_update_data( job_id: &Uuid, log_offset: Option, stream_offset: Option, - get_progress: Option, + get_progress: bool, running: Option, log_view: bool, get_full_job_on_completion: bool, only_result: Option, no_logs: Option, + is_flow: Option, + flow_stream_job_id: Option, ) -> error::Result { let tags = if log_view { log_job_view( @@ -6562,128 +7141,200 @@ async fn get_job_update_data( None }; + let ignore_flow_stream_job_id = is_flow.is_some_and(|x| !x) || flow_stream_job_id.is_some(); + if only_result.unwrap_or(false) { - let result = if let Some(tags) = tags { - let r = - sqlx::query!( - "SELECT result as \"result: sqlx::types::Json>\", v2_job.tag, - v2_job_queue.running as \"running: Option\", SUBSTR(rs.stream, $3) AS \"result_stream: Option\", CHAR_LENGTH(rs.stream) AS stream_offset + let (result, running, mut result_stream, mut new_stream_offset, new_flow_stream_job_id) = + if let Some(tags) = tags { + let r = sqlx::query!( + " + WITH result_stream AS ( + SELECT + string_agg(stream, '' order by idx asc) as stream, + job_id, + max(idx) + 1 as offset + FROM job_result_stream_v2 + WHERE job_id = $2 AND idx >= $3 + GROUP BY job_id + ) + SELECT + jc.result as \"result: sqlx::types::Json>\", + v2_job.tag, + v2_job_queue.running as \"running: Option\", + rs.stream AS \"result_stream: Option\", + rs.offset AS stream_offset, + CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job FROM v2_job LEFT JOIN v2_job_queue USING (id) - LEFT JOIN v2_job_completed USING (id) - LEFT JOIN job_result_stream rs ON rs.job_id = $2 + LEFT JOIN v2_job_completed jc USING (id) + LEFT JOIN v2_job_status js USING (id) + LEFT JOIN result_stream rs ON rs.job_id = $2 WHERE v2_job.id = $2 AND v2_job.workspace_id = $1", - w_id, - job_id, - stream_offset.unwrap_or(0), - ) - .fetch_optional(db) - .await? - .ok_or_else(|| Error::NotFound(format!("Job not found: {}", job_id)))?; - - if !tags.contains(&r.tag.as_str()) { - return Err(Error::NotAuthorized(format!( - "Job tag {} is not in the scope tags: {}", - r.tag, - tags.join(", ") - ))); - } - let running = r.running.as_ref().map(|x| *x); - ( - r.result.map(|x| x.0), - running, - r.result_stream.flatten(), - r.stream_offset, + w_id, + job_id, + stream_offset.unwrap_or(0), + ignore_flow_stream_job_id, ) - } else { - if running.is_some_and(|x| !x) { - let r = sqlx::query!( - "SELECT + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("Job not found: {}", job_id)))?; + + if !tags.contains(&r.tag.as_str()) { + return Err(Error::NotAuthorized(format!( + "Job tag {} is not in the scope tags: {}", + r.tag, + tags.join(", ") + ))); + } + let running = r.running.as_ref().map(|x| *x); + ( + r.result.map(|x| x.0), + running, + r.result_stream.flatten(), + r.stream_offset, + r.stream_job, + ) + } else { + if running.is_some_and(|x| !x) { + let r = sqlx::query!( + " + WITH result_stream AS ( + SELECT + string_agg(stream, '' order by idx asc) as stream, + job_id, + max(idx) + 1 as offset + FROM job_result_stream_v2 + WHERE job_id = $1 AND idx >= $3 + GROUP BY job_id + ) + SELECT COALESCE(jc.result, jc.result) as \"result: sqlx::types::Json>\", jq.running as \"running: Option\", - SUBSTR(rs.stream, $3) AS \"result_stream: Option\", - CHAR_LENGTH(rs.stream) + 1 AS stream_offset + rs.stream AS \"result_stream: Option\", + rs.offset AS stream_offset, + CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job FROM ( SELECT $1::uuid as job_id, $2::text as workspace_id ) base LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id LEFT JOIN v2_job_queue jq ON jq.id = base.job_id AND jq.workspace_id = base.workspace_id - LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id + LEFT JOIN v2_job_status js ON js.id = base.job_id + LEFT JOIN result_stream rs ON rs.job_id = base.job_id WHERE base.job_id = $1", job_id, w_id, stream_offset.unwrap_or(0), + ignore_flow_stream_job_id, ).fetch_optional(db).await?; - if let Some(r) = r { - let running = r.running.as_ref().map(|x| *x); - ( - r.result.map(|x| x.0), - running, - r.result_stream.flatten(), - r.stream_offset, - ) + if let Some(r) = r { + let running = r.running.as_ref().map(|x| *x); + ( + r.result.map(|x| x.0), + running, + r.result_stream.flatten(), + r.stream_offset, + r.stream_job, + ) + } else { + (None, None, None, None, None) + } } else { - (None, None, None, None) - } - } else { - let q = sqlx::query!( - "SELECT + let q = sqlx::query!( + " + WITH result_stream AS ( + SELECT + string_agg(stream, '' order by idx asc) as stream, + job_id, + max(idx) + 1 as offset + FROM job_result_stream_v2 + WHERE job_id = $2 AND idx >= $3 + GROUP BY job_id + ) + SELECT COALESCE(jc.result, NULL) as \"result: sqlx::types::Json>\", - SUBSTR(rs.stream, $3) AS \"result_stream: Option\", - CHAR_LENGTH(rs.stream) + 1 AS stream_offset + rs.stream AS \"result_stream: Option\", + rs.offset AS stream_offset, + COALESCE(js.flow_status, jc.flow_status) as \"flow_status: sqlx::types::Json>\", + CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job FROM ( SELECT $2::uuid as job_id, $1::text as workspace_id ) base LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id - LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id + LEFT JOIN v2_job_status js ON js.id = base.job_id + LEFT JOIN result_stream rs ON rs.job_id = base.job_id WHERE base.job_id = $2", w_id, job_id, stream_offset.unwrap_or(0), + ignore_flow_stream_job_id, ) .fetch_optional(db) .await?; - if let Some(r) = q { - ( - r.result.map(|x| x.0), - running, - r.result_stream.flatten(), - r.stream_offset, - ) - } else { - (None, None, None, None) + if let Some(r) = q { + ( + r.result.map(|x| x.0), + running, + r.result_stream.flatten(), + r.stream_offset, + r.stream_job, + ) + } else { + (None, None, None, None, None) + } } - } - }; + }; + + let flow_stream_job_id = flow_stream_job_id.or(new_flow_stream_job_id); + + let flow_stream_delta = + get_flow_stream_delta(db, flow_stream_job_id, stream_offset).await?; + + if let Some((flow_result_stream, flow_stream_offset)) = flow_stream_delta { + result_stream = flow_result_stream; + new_stream_offset = flow_stream_offset; + } + Ok(JobUpdate { - running: result.1, - completed: if result.0.is_some() { Some(true) } else { None }, + running, + completed: if result.is_some() { Some(true) } else { None }, log_offset: None, new_logs: None, - new_result_stream: result.2, - stream_offset: result.3, + new_result_stream: result_stream, + stream_offset: new_stream_offset, mem_peak: None, progress: None, job: None, flow_status: None, workflow_as_code_status: None, - only_result: result.0, + only_result: result, + flow_stream_job_id, }) } else { - let record = sqlx::query!( - "SELECT + let mut record = sqlx::query!( + " + WITH result_stream AS ( + SELECT + string_agg(stream, '' order by idx asc) as stream, + job_id, + max(idx) + 1 as offset + FROM job_result_stream_v2 + WHERE job_id = $3 AND idx >= $8 + GROUP BY job_id + ) + SELECT c.id IS NOT NULL AS completed, CASE WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END) ELSE false END AS running, CASE WHEN $7::BOOLEAN THEN NULL ELSE SUBSTR(logs, GREATEST($1 - log_offset, 0)) END AS logs, - SUBSTR(rs.stream, $8) AS new_result_stream, + rs.stream AS new_result_stream, COALESCE(r.memory_peak, c.memory_peak) AS mem_peak, COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json>\", + (COALESCE(c.flow_status, f.flow_status)->>'stream_job')::uuid AS stream_job, COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json>\", CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset, - CHAR_LENGTH(rs.stream) + 1 AS stream_offset, + rs.offset AS stream_offset, created_by AS \"created_by!\", CASE WHEN $4::BOOLEAN THEN ( SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc' @@ -6694,14 +7345,14 @@ async fn get_job_update_data( LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status f USING (id) LEFT JOIN v2_job_completed c USING (id) - LEFT JOIN job_result_stream rs ON rs.job_id = $3 + LEFT JOIN result_stream rs ON rs.job_id = $3 LEFT JOIN job_logs ON job_logs.job_id = $3 WHERE j.workspace_id = $2 AND j.id = $3 AND ($6::text[] IS NULL OR j.tag = ANY($6))", log_offset, w_id, job_id, - get_progress.unwrap_or(false), + get_progress, running, tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, no_logs.unwrap_or(false), @@ -6718,12 +7369,25 @@ async fn get_job_update_data( } let job = if record.completed.unwrap_or(false) && get_full_job_on_completion { - let get = GetQuery::new().with_auth(&opt_authed).without_logs(); + let get = GetQuery::new() + .with_auth(&opt_authed) + .without_logs() + .without_code(); Some(get.fetch(&db, job_id, &w_id).await?) } else { None }; + let flow_stream_job_id = flow_stream_job_id.or(record.stream_job); + + let flow_stream_delta = + get_flow_stream_delta(db, flow_stream_job_id, stream_offset).await?; + + if let Some((new_result_stream, stream_offset)) = flow_stream_delta { + record.new_result_stream = new_result_stream; + record.stream_offset = stream_offset; + } + Ok(JobUpdate { running: record.running, completed: record.completed, @@ -6741,6 +7405,7 @@ async fn get_job_update_data( .flow_status .map(|x: sqlx::types::Json>| x.0), only_result: None, + flow_stream_job_id, }) } } @@ -6757,7 +7422,7 @@ pub fn filter_list_completed_query( if join_outstanding_wait_times { sqlb.left() .join("outstanding_wait_time") - .on_eq("v2_job.id", "outstanding_wait_time.job_id"); + .on_eq("v2_job_completed.id", "outstanding_wait_time.job_id"); } if let Some(label) = &lq.label { @@ -6785,7 +7450,8 @@ pub fn filter_list_completed_query( } if w_id != "admins" || !lq.all_workspaces.is_some_and(|x| x) { - sqlb.and_where_eq("v2_job.workspace_id", "?".bind(&w_id)); + sqlb.and_where_eq("v2_job_completed.workspace_id", "?".bind(&w_id)) + .and_where_eq("v2_job.workspace_id", "?".bind(&w_id)); } if let Some(p) = &lq.schedule_path { @@ -6836,8 +7502,12 @@ pub fn filter_list_completed_query( sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339())); } if let Some(dt) = &lq.created_or_started_after { - sqlb.and_where_ge("created_at", "?".bind(&dt.to_rfc3339())); - sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339())); + let ts = dt.to_rfc3339(); + sqlb.and_where(format!( + "(created_at >= '{}' OR started_at >= '{}')", + ts.replace("'", "''"), + ts.replace("'", "''") + )); } if let Some(dt) = &lq.created_before { @@ -6851,6 +7521,13 @@ pub fn filter_list_completed_query( sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339())); } + if let Some(dt) = &lq.completed_after { + sqlb.and_where_ge("completed_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(dt) = &lq.completed_before { + sqlb.and_where_le("completed_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(sk) = &lq.is_skipped { if *sk { sqlb.and_where_eq("status", "'skipped'"); @@ -6886,8 +7563,7 @@ pub fn filter_list_completed_query( } if lq.is_not_schedule.unwrap_or(false) { - sqlb.and_where("trigger_kind != 'schedule'") - .or_where("trigger_kind IS NULL"); + sqlb.and_where("trigger_kind IS DISTINCT FROM 'schedule'"); } sqlb @@ -6904,7 +7580,14 @@ pub fn list_completed_jobs_query( ) -> SqlBuilder { let mut sqlb = SqlBuilder::select_from("v2_job_completed") .fields(fields) - .order_by("v2_job.created_at", lq.order_desc.unwrap_or(true)) + .order_by( + if lq.completed_before.is_some() || lq.completed_after.is_some() { + "v2_job_completed.completed_at" + } else { + "v2_job.created_at" + }, + lq.order_desc.unwrap_or(true), + ) .offset(offset) .clone(); if let Some(per_page) = per_page { @@ -6933,6 +7616,10 @@ pub struct ListCompletedQuery { pub created_or_started_before: Option>, pub created_or_started_after: Option>, pub created_or_started_after_completed_jobs: Option>, + pub created_before_queue: Option>, + pub created_after_queue: Option>, + pub completed_after: Option>, + pub completed_before: Option>, pub success: Option, pub running: Option, pub parent_job: Option, @@ -6972,8 +7659,8 @@ async fn list_completed_jobs( offset, &lq, &[ - "v2_job.id", - "v2_job.workspace_id", + "v2_job_completed.id", + "v2_job_completed.workspace_id", "v2_job.parent_job", "v2_job.created_by", "v2_job.created_at", @@ -7188,10 +7875,10 @@ async fn count_by_tag( let counts = sqlx::query_as!( TagCount, r#" - SELECT tag as "tag!", COUNT(*) as "count!" - FROM v2_as_completed_job - WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2) - GROUP BY tag + SELECT j.tag as "tag!", COUNT(*) as "count!" + FROM v2_job_completed c JOIN v2_job j USING (id) + WHERE c.started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR j.workspace_id = $2) + GROUP BY j.tag ORDER BY "count!" DESC "#, horizon as f64, diff --git a/backend/windmill-api/src/kafka_triggers_oss.rs b/backend/windmill-api/src/kafka_triggers_oss.rs deleted file mode 100644 index 4227023648..0000000000 --- a/backend/windmill-api/src/kafka_triggers_oss.rs +++ /dev/null @@ -1,55 +0,0 @@ -#[cfg(feature = "private")] -#[allow(unused)] -pub use crate::kafka_triggers_ee::*; - -#[cfg(not(feature = "private"))] -use crate::db::DB; -#[cfg(not(feature = "private"))] -use serde::{Deserialize, Serialize}; - -#[derive(Serialize, Deserialize)] -#[cfg(not(feature = "private"))] -pub struct KafkaResourceSecurity {} - -#[cfg(not(feature = "private"))] -pub fn start_kafka_consumers( - _db: DB, - mut _killpill_rx: tokio::sync::broadcast::Receiver<()>, -) -> () { - // implementation is not open source -} - -#[derive(Serialize, Deserialize)] -#[cfg(not(feature = "private"))] -pub enum KafkaTriggerConfigConnection {} - -#[derive(Serialize, Clone)] -#[cfg(not(feature = "private"))] -pub struct KafkaTrigger { - pub workspace_id: String, - pub path: String, - pub kafka_resource_path: String, - pub group_id: String, - pub topics: Vec, - pub script_path: String, - pub is_flow: bool, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - #[serde(skip_serializing_if = "Option::is_none")] - pub server_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_server_ping: Option>, - pub extra_perms: serde_json::Value, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option< - sqlx::types::Json>>, - >, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, -} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index dd9a6b7705..802f6f0353 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -20,8 +20,10 @@ use crate::smtp_server_oss::SmtpServer; #[cfg(feature = "mcp")] use crate::mcp::{extract_and_store_workspace_id, setup_mcp_server, shutdown_mcp_server}; +use crate::triggers::start_all_listeners; #[cfg(feature = "mcp")] use rmcp::transport::streamable_http_server::session::local::LocalSessionManager; +use tower_http::catch_panic::CatchPanicLayer; use crate::tracing_init::MyOnFailure; use crate::{ @@ -60,7 +62,7 @@ use tower_http::{ }; use windmill_common::db::UserDB; use windmill_common::worker::CLOUD_HOSTED; -use windmill_common::{utils::GIT_VERSION, BASE_URL, INSTANCE_NAME}; +use windmill_common::{utils::{configure_client, GIT_VERSION}, BASE_URL, INSTANCE_NAME}; use crate::scim_oss::has_scim_token; use windmill_common::error::AppError; @@ -85,7 +87,8 @@ pub mod ee; pub mod ee_oss; pub mod embeddings; mod favorite; -mod flows; +mod flow_conversations; +pub mod flows; mod folders; mod granular_acls; mod groups; @@ -98,25 +101,17 @@ mod inkeep_oss; mod inputs; mod integration; mod live_migrations; -#[cfg(feature = "postgres_trigger")] -mod postgres_triggers; +#[cfg(feature = "http_trigger")] +mod openapi; #[cfg(all(feature = "private", feature = "parquet"))] pub mod s3_proxy_ee; mod s3_proxy_oss; -mod trigger_helpers; - -pub mod openapi; - mod approvals; #[cfg(all(feature = "enterprise", feature = "private"))] pub mod apps_ee; #[cfg(feature = "enterprise")] mod apps_oss; -#[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] -pub mod gcp_triggers_ee; -#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] -mod gcp_triggers_oss; #[cfg(all(feature = "enterprise", feature = "private"))] pub mod git_sync_ee; #[cfg(feature = "enterprise")] @@ -127,16 +122,6 @@ pub mod job_helpers_ee; mod job_helpers_oss; pub mod job_metrics; pub mod jobs; -#[cfg(all(feature = "enterprise", feature = "kafka", feature = "private"))] -pub mod kafka_triggers_ee; -#[cfg(all(feature = "enterprise", feature = "kafka"))] -mod kafka_triggers_oss; -#[cfg(feature = "mqtt_trigger")] -mod mqtt_triggers; -#[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] -pub mod nats_triggers_ee; -#[cfg(all(feature = "enterprise", feature = "nats"))] -mod nats_triggers_oss; #[cfg(all(feature = "oauth2", feature = "private"))] pub mod oauth2_ee; #[cfg(feature = "oauth2")] @@ -162,10 +147,6 @@ mod slack_approvals; pub mod smtp_server_ee; #[cfg(feature = "smtp")] mod smtp_server_oss; -#[cfg(all(feature = "enterprise", feature = "sqs_trigger", feature = "private"))] -pub mod sqs_triggers_ee; -#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] -mod sqs_triggers_oss; #[cfg(feature = "private")] pub mod teams_approvals_ee; mod teams_approvals_oss; @@ -176,6 +157,9 @@ pub mod stripe_ee; #[cfg(all(feature = "stripe", feature = "enterprise"))] mod stripe_oss; #[cfg(feature = "private")] +pub mod teams_cache_ee; +mod teams_cache_oss; +#[cfg(feature = "private")] pub mod teams_ee; mod teams_oss; mod token; @@ -189,8 +173,6 @@ mod utils; pub mod var_resource_cache; mod variables; pub mod webhook_util; -#[cfg(feature = "websocket")] -mod websocket_triggers; mod workers; mod workspaces; #[cfg(feature = "private")] @@ -202,6 +184,7 @@ mod workspaces_oss; #[cfg(feature = "mcp")] mod mcp; +pub use apps::EditApp; pub const DEFAULT_BODY_LIMIT: usize = 2097152 * 100; // 200MB lazy_static::lazy_static! { @@ -216,11 +199,11 @@ lazy_static::lazy_static! { pub static ref IS_SECURE: Arc> = Arc::new(RwLock::new(false)); - pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new() + pub static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() .user_agent("windmill/beta") .connect_timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(30)) - .danger_accept_invalid_certs(std::env::var("ACCEPT_INVALID_CERTS").is_ok()) + .danger_accept_invalid_certs(std::env::var("ACCEPT_INVALID_CERTS").is_ok())) .build().unwrap(); @@ -389,47 +372,7 @@ pub async fn run_server( let triggers_service = triggers::generate_trigger_routers(); if !*CLOUD_HOSTED && server_mode && !mcp_mode { - #[cfg(feature = "websocket")] - { - let ws_killpill_rx = killpill_rx.resubscribe(); - websocket_triggers::start_websockets(db.clone(), ws_killpill_rx); - } - - #[cfg(all(feature = "enterprise", feature = "kafka"))] - { - let kafka_killpill_rx = killpill_rx.resubscribe(); - kafka_triggers_oss::start_kafka_consumers(db.clone(), kafka_killpill_rx); - } - - #[cfg(all(feature = "enterprise", feature = "nats"))] - { - let nats_killpill_rx = killpill_rx.resubscribe(); - nats_triggers_oss::start_nats_consumers(db.clone(), nats_killpill_rx); - } - - #[cfg(feature = "postgres_trigger")] - { - let db_killpill_rx = killpill_rx.resubscribe(); - postgres_triggers::start_database(db.clone(), db_killpill_rx); - } - - #[cfg(feature = "mqtt_trigger")] - { - let mqtt_killpill_rx = killpill_rx.resubscribe(); - mqtt_triggers::start_mqtt_consumer(db.clone(), mqtt_killpill_rx); - } - - #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] - { - let sqs_killpill_rx = killpill_rx.resubscribe(); - sqs_triggers_oss::start_sqs(db.clone(), sqs_killpill_rx); - } - - #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] - { - let gcp_killpill_rx = killpill_rx.resubscribe(); - gcp_triggers_oss::start_consuming_gcp_pubsub_event(db.clone(), gcp_killpill_rx); - } + start_all_listeners(db.clone(), &killpill_rx); } let listener = tokio::net::TcpListener::bind(addr) @@ -470,7 +413,7 @@ pub async fn run_server( }; #[cfg(feature = "agent_worker_server")] - let (agent_workers_router, agent_workers_bg_processor, agent_workers_killpill_tx) = + let (agent_workers_router, agent_workers_bg_processor, agent_workers_job_completed_tx) = if server_mode { agent_workers_oss::workspaced_service(db.clone(), _base_internal_url.clone()) } else { @@ -502,6 +445,10 @@ pub async fn run_server( .nest("/drafts", drafts::workspaced_service()) .nest("/favorites", favorite::workspaced_service()) .nest("/flows", flows::workspaced_service()) + .nest( + "/flow_conversations", + flow_conversations::workspaced_service(), + ) .nest("/folders", folders::workspaced_service()) .nest("/groups", groups::workspaced_service()) .nest("/inputs", inputs::workspaced_service()) @@ -529,7 +476,17 @@ pub async fn run_server( .nest("/variables", variables::workspaced_service()) .nest("/workspaces", workspaces::workspaced_service()) .nest("/oidc", oidc_oss::workspaced_service()) - .nest("/openapi", openapi::openapi_service()) + .nest("/openapi", { + #[cfg(feature = "http_trigger")] + { + openapi::openapi_service() + } + + #[cfg(not(feature = "http_trigger"))] + { + Router::new() + } + }) .merge(triggers_service), ) .nest("/workspaces", workspaces::global_service()) @@ -594,7 +551,14 @@ pub async fn run_server( .nest("/agent_workers", { #[cfg(feature = "agent_worker_server")] { - agent_workers_oss::global_service().layer(Extension(agent_cache.clone())) + if let Some(agent_workers_job_completed_tx) = + agent_workers_job_completed_tx.clone() + { + agent_workers_oss::global_service(agent_workers_job_completed_tx) + .layer(Extension(agent_cache.clone())) + } else { + Router::new() + } } #[cfg(not(feature = "agent_worker_server"))] { @@ -735,10 +699,18 @@ pub async fn run_server( ) }; + let app = app.layer(CatchPanicLayer::custom(|err| { + tracing::error!("panic in handler, returning 500: {:?}", err); + Response::builder() + .status(http::StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::from("Internal Server Error")) + .unwrap() + })); + if let Some(name) = name.as_ref() { tracing::info!("server starting for name={name}"); } - let server = axum::serve(listener, app.into_make_service()); + let server = axum::serve(listener, app.into_make_service()).tcp_nodelay(!server_mode); tracing::info!( instance = %*INSTANCE_NAME, @@ -748,15 +720,16 @@ pub async fn run_server( name.map(|x| format!("name={x}")).unwrap_or_default() ); - port_tx - .send(format!("http://localhost:{}", port)) - .expect("Failed to send port"); + if let Err(e) = port_tx.send(format!("http://localhost:{}", port)) { + tracing::error!("Failed to send port: {e:#}"); + return Err(anyhow::anyhow!("Failed to send port, exiting early: {e:#}")); + } let server = server.with_graceful_shutdown(async move { killpill_rx.recv().await.ok(); #[cfg(feature = "agent_worker_server")] - if let Some(agent_workers_killpill_tx) = agent_workers_killpill_tx { - if let Err(e) = agent_workers_killpill_tx.kill().await { + if let Some(agent_workers_job_completed_tx) = agent_workers_job_completed_tx { + if let Err(e) = agent_workers_job_completed_tx.kill().await { tracing::error!("Error killing agent workers: {e:#}"); } } @@ -846,8 +819,11 @@ async fn openapi_json() -> Response { .unwrap() } -pub async fn migrate_db(db: &DB) -> anyhow::Result>> { - db::migrate(db) +pub async fn migrate_db( + db: &DB, + killpill_rx: tokio::sync::broadcast::Receiver<()>, +) -> anyhow::Result>> { + db::migrate(db, killpill_rx) .await .map_err(|e| anyhow::anyhow!("Error migrating db: {e:#}")) } diff --git a/backend/windmill-api/src/live_migrations.rs b/backend/windmill-api/src/live_migrations.rs index 0e3fb97e02..18e05967b0 100644 --- a/backend/windmill-api/src/live_migrations.rs +++ b/backend/windmill-api/src/live_migrations.rs @@ -447,19 +447,6 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> { i += 1; tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query!("create index concurrently if not exists ix_completed_job_workspace_id_started_at_new_2 ON v2_job_completed (workspace_id, started_at DESC)") - .execute(db) - .await?; - i += 1; - tracing::info!("step {i} of {migration_job_name} migration"); - - sqlx::query!("create index concurrently if not exists ix_job_root_job_index_by_path_2 ON v2_job (workspace_id, runnable_path, created_at desc) WHERE parent_job IS NULL") - .execute(db) - .await?; - - i += 1; - tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS root_job_index_by_path_2") .execute(db) .await?; @@ -467,13 +454,6 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> { i += 1; tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query!("create index concurrently if not exists ix_job_created_at ON v2_job (created_at DESC)") - .execute(db) - .await?; - - i += 1; - tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query( "DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_created_at_new_2", ) @@ -523,9 +503,9 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> { }); run_windmill_migration!("v2_improve_v2_job_indices_ii", &db, |tx| { - sqlx::query!("create index concurrently if not exists ix_v2_job_workspace_id_created_at ON v2_job (workspace_id, created_at DESC) where kind in ('script', 'flow', 'singlescriptflow') AND parent_job IS NULL") - .execute(db) - .await?; + sqlx::query!("create index concurrently if not exists ix_v2_job_workspace_id_created_at ON v2_job (workspace_id, created_at DESC) where kind in ('script', 'flow', 'singlestepflow') AND parent_job IS NULL") + .execute(db) + .await?; sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS ix_job_workspace_id_created_at_new_6") .execute(db) @@ -609,5 +589,52 @@ WHERE operation IN ('users.login', 'oauth.login', 'users.token.refresh');" .execute(db) .await?; }); + + run_windmill_migration!("v2_job_completed_completed_at_9", db, |tx| { + let migration_job_name = "v2_job_completed_completed_at"; + let mut i = 1; + tracing::info!("step {i} of {migration_job_name} migration"); + sqlx::query!("create index concurrently if not exists ix_job_workspace_id_completed_at_all ON v2_job_completed (workspace_id, completed_at DESC)") + .execute(db) + .await?; + i += 1; + + sqlx::query!("CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_job_v2_job_root_by_path_2 ON v2_job (workspace_id, runnable_path) WHERE parent_job IS NULL;") + .execute(db) + .await?; + + i += 1; + tracing::info!("step {i} of {migration_job_name} migration"); + + sqlx::query!("create index concurrently if not exists ix_job_root_job_index_by_path_2 ON v2_job (workspace_id, runnable_path, created_at desc) WHERE parent_job IS NULL") + .execute(db) + .await?; + + i += 1; + tracing::info!("step {i} of {migration_job_name} migration"); + + sqlx::query!( + "DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_started_at_new_2" + ) + .execute(db) + .await?; + + i += 1; + tracing::info!("step {i} of {migration_job_name} migration"); + + sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS ix_job_created_at") + .execute(db) + .await?; + + i += 1; + + sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS ix_v2_job_root_by_path") + .execute(db) + .await?; + + i += 1; + tracing::info!("step {i} of {migration_job_name} migration"); + }); + Ok(()) } diff --git a/backend/windmill-api/src/mcp/server.rs b/backend/windmill-api/src/mcp/server.rs index e51d008e3c..8a09459202 100644 --- a/backend/windmill-api/src/mcp/server.rs +++ b/backend/windmill-api/src/mcp/server.rs @@ -4,16 +4,17 @@ //! specification. This is a thin orchestration layer that delegates to the appropriate //! modules for tool management, database operations, and schema transformation. -use std::borrow::Cow; use std::collections::HashMap; use std::sync::Arc; +use std::{borrow::Cow, time::Duration}; use axum::body::to_bytes; use rmcp::{ handler::server::ServerHandler, model::*, service::{RequestContext, RoleServer}, - Error, + transport::StreamableHttpServerConfig, + ErrorData, }; use serde_json::Value; use tokio::try_join; @@ -26,35 +27,29 @@ use crate::jobs::{ run_wait_result_flow_by_path_internal, run_wait_result_script_by_path_internal, RunJobQuery, }; +use super::tools::endpoint_tools::{ + all_endpoint_tools, call_endpoint_tool, endpoint_tools_to_mcp_tools, EndpointTool, +}; use super::utils::{ database::{ - check_scopes, get_items, get_resources_types, get_scripts_from_hub, get_item_schema, get_hub_script_schema + check_scopes, get_hub_script_schema, get_item_schema, get_items, get_resources_types, + get_scripts_from_hub, + }, + models::{ + FlowInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, ToolableItem, WorkspaceId, }, - models::{ScriptInfo, FlowInfo, ResourceInfo, ResourceType, SchemaType, ToolableItem, WorkspaceId}, schema::transform_schema_for_resources, transform::{reverse_transform, reverse_transform_key}, }; -use super::tools::{ - endpoint_tools::{all_endpoint_tools, endpoint_tools_to_mcp_tools, call_endpoint_tool, EndpointTool}, -}; use axum::{ - extract::Path, - http::Request, - middleware::Next, - response::Response, - routing::get, - Json, - Router, + extract::Path, http::Request, middleware::Next, response::Response, routing::get, Json, Router, }; use rmcp::transport::streamable_http_server::{ - session::local::LocalSessionManager, - SessionManager, - StreamableHttpService, + session::local::LocalSessionManager, SessionManager, StreamableHttpService, }; use windmill_common::error::JsonResult; - /// MCP Server Runner - implements the core MCP protocol handlers #[derive(Clone)] pub struct Runner {} @@ -72,7 +67,7 @@ impl Runner { workspace_id: &str, resources_cache: &mut HashMap>, resources_types: &Vec, - ) -> Result { + ) -> Result { let is_hub = item.is_hub(); let path = item.get_path_or_id(); let item_type = item.item_type(); @@ -120,52 +115,73 @@ impl Runner { name: Cow::Owned(path), description: Some(Cow::Owned(description)), input_schema: Arc::new(input_schema_map), + title: Some(item.get_summary().to_string()), + output_schema: None, + icons: None, annotations: Some(ToolAnnotations { title: Some(item.get_summary().to_string()), - read_only_hint: Some(false), // Can modify environment + read_only_hint: Some(false), // Can modify environment destructive_hint: Some(true), // Can potentially be destructive idempotent_hint: Some(false), // Are not guaranteed to be idempotent - open_world_hint: Some(true), // Can interact with external services + open_world_hint: Some(true), // Can interact with external services }), }) } } - impl ServerHandler for Runner { /// Handles the `CallTool` request from the MCP client async fn call_tool( &self, request: CallToolRequestParam, context: RequestContext, - ) -> Result { + ) -> Result { let http_parts = context .extensions .get::() .ok_or_else(|| { tracing::error!("http::request::Parts not found"); - Error::internal_error("http::request::Parts not found", None) + ErrorData::internal_error("http::request::Parts not found", None) })?; let authed = http_parts.extensions.get::().ok_or_else(|| { tracing::error!("ApiAuthed Axum extension not found"); - Error::internal_error("ApiAuthed Axum extension not found", None) + ErrorData::internal_error("ApiAuthed Axum extension not found", None) })?; check_scopes(authed)?; + if request.name.ends_with("_TRUNC") { + return Ok(CallToolResult::error( + vec![ + Annotated::new( + RawContent::Text(RawTextContent { + text: + "Tool path is too long. Consider shortening it to make it compatible with MCP." + .to_string(), + meta: None, + }), + None + ), + ] + )); + } + let db = http_parts.extensions.get::().ok_or_else(|| { tracing::error!("DB Axum extension not found"); - Error::internal_error("DB Axum extension not found", None) + ErrorData::internal_error("DB Axum extension not found", None) })?; let user_db = http_parts.extensions.get::().ok_or_else(|| { tracing::error!("UserDB Axum extension not found"); - Error::internal_error("UserDB Axum extension not found", None) + ErrorData::internal_error("UserDB Axum extension not found", None) })?; - + let args = request.arguments.map(Value::Object).ok_or_else(|| { - Error::invalid_params("Missing arguments for tool", Some(request.name.clone().into())) + ErrorData::invalid_params( + "Missing arguments for tool", + Some(request.name.clone().into()), + ) })?; let workspace_id = http_parts @@ -173,7 +189,7 @@ impl ServerHandler for Runner { .get::() .ok_or_else(|| { tracing::error!("WorkspaceId not found"); - Error::internal_error("WorkspaceId not found", None) + ErrorData::internal_error("WorkspaceId not found", None) }) .map(|w_id| w_id.0.clone())?; @@ -182,18 +198,20 @@ impl ServerHandler for Runner { for endpoint_tool in endpoint_tools { if endpoint_tool.name.as_ref() == request.name { // This is an endpoint tool, forward to the actual HTTP endpoint - let result = call_endpoint_tool(&endpoint_tool, args.clone(), &workspace_id, &authed).await?; + let result = + call_endpoint_tool(&endpoint_tool, args.clone(), &workspace_id, &authed) + .await?; return Ok(CallToolResult::success(vec![Content::text( - serde_json::to_string_pretty(&result).unwrap_or_else(|_| "{}".to_string()) + serde_json::to_string_pretty(&result).unwrap_or_else(|_| "{}".to_string()), )])); } } // Continue with script/flow logic let (tool_type, path, is_hub) = reverse_transform(&request.name).map_err(|e| { - Error::internal_error(format!("Failed to reverse transform path: {}", e), None) + ErrorData::internal_error(format!("Failed to reverse transform path: {}", e), None) })?; - + let item_schema = if is_hub { get_hub_script_schema(&format!("hub/{}", path), db).await? } else { @@ -259,14 +277,20 @@ impl ServerHandler for Runner { let body_bytes = to_bytes(response.into_body(), usize::MAX) .await .map_err(|e| { - Error::internal_error(format!("Failed to read response body: {}", e), None) + ErrorData::internal_error( + format!("Failed to read response body: {}", e), + None, + ) })?; let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| { - Error::internal_error(format!("Failed to decode response body: {}", e), None) + ErrorData::internal_error( + format!("Failed to decode response body: {}", e), + None, + ) })?; Ok(CallToolResult::success(vec![Content::text(body_str)])) } - Err(e) => Err(Error::internal_error( + Err(e) => Err(ErrorData::internal_error( format!("Failed to run script: {}", e), None, )), @@ -278,30 +302,30 @@ impl ServerHandler for Runner { &self, _request: Option, mut _context: RequestContext, - ) -> Result { + ) -> Result { let http_parts = _context .extensions .get::() .ok_or_else(|| { tracing::error!("http::request::Parts not found"); - Error::internal_error("http::request::Parts not found", None) + ErrorData::internal_error("http::request::Parts not found", None) })?; let authed = http_parts.extensions.get::().ok_or_else(|| { tracing::error!("ApiAuthed Axum extension not found"); - Error::internal_error("ApiAuthed Axum extension not found", None) + ErrorData::internal_error("ApiAuthed Axum extension not found", None) })?; check_scopes(authed)?; let db = http_parts.extensions.get::().ok_or_else(|| { tracing::error!("DB Axum extension not found"); - Error::internal_error("DB Axum extension not found", None) + ErrorData::internal_error("DB Axum extension not found", None) })?; let user_db = http_parts.extensions.get::().ok_or_else(|| { tracing::error!("UserDB Axum extension not found"); - Error::internal_error("UserDB Axum extension not found", None) + ErrorData::internal_error("UserDB Axum extension not found", None) })?; let workspace_id = http_parts @@ -309,7 +333,7 @@ impl ServerHandler for Runner { .get::() .ok_or_else(|| { tracing::error!("WorkspaceId not found"); - Error::internal_error("WorkspaceId not found", None) + ErrorData::internal_error("WorkspaceId not found", None) }) .map(|w_id| w_id.0.clone())?; @@ -319,10 +343,18 @@ impl ServerHandler for Runner { .iter() .find(|scope| scope.starts_with("mcp:") && !scope.contains("hub")) }); - let hub_scope = scopes.and_then(|scopes| scopes.iter().find(|scope| scope.starts_with("mcp:hub"))); + let hub_scope = + scopes.and_then(|scopes| scopes.iter().find(|scope| scope.starts_with("mcp:hub"))); let (scope_type, scope_path) = owned_scope.map_or(("all", None), |scope| { let parts = scope.split(":").collect::>(); - (parts[1], if parts.len() == 3 { Some(parts[2]) } else { None }) + ( + parts[1], + if parts.len() == 3 { + Some(parts[2]) + } else { + None + }, + ) }); let scope_integrations = hub_scope.and_then(|scope| { let parts = scope.split(":").collect::>(); @@ -341,8 +373,14 @@ impl ServerHandler for Runner { "script", scope_path.as_deref(), ); - let flows_fn = - get_items::(user_db, authed, &workspace_id, scope_type, "flow", scope_path.as_deref()); + let flows_fn = get_items::( + user_db, + authed, + &workspace_id, + scope_type, + "flow", + scope_path.as_deref(), + ); let resources_types_fn = get_resources_types(user_db, authed, &workspace_id); let hub_scripts_fn = get_scripts_from_hub(db, scope_integrations.as_deref()); let (scripts, flows, resources_types, hub_scripts) = if scope_integrations.is_some() { @@ -410,10 +448,9 @@ impl ServerHandler for Runner { fn get_info(&self) -> ServerInfo { ServerInfo { - protocol_version: Default::default(), + protocol_version: ProtocolVersion::default(), capabilities: ServerCapabilities::builder() .enable_tools() - .enable_tool_list_changed() .build(), server_info: Implementation::from_build_env(), instructions: Some("This server provides a list of scripts and flows the user can run on Windmill. Each flow and script is a tool callable with their respective arguments.".to_string()), @@ -424,7 +461,7 @@ impl ServerHandler for Runner { &self, _request: InitializeRequestParam, _context: RequestContext, - ) -> Result { + ) -> Result { Ok(self.get_info()) } @@ -432,7 +469,7 @@ impl ServerHandler for Runner { &self, _request: Option, _context: RequestContext, - ) -> Result { + ) -> Result { Ok(ListResourcesResult { resources: vec![], next_cursor: None }) } @@ -440,7 +477,7 @@ impl ServerHandler for Runner { &self, _request: Option, _context: RequestContext, - ) -> Result { + ) -> Result { Ok(ListPromptsResult::default()) } @@ -448,7 +485,7 @@ impl ServerHandler for Runner { &self, _request: Option, _context: RequestContext, - ) -> Result { + ) -> Result { Ok(ListResourceTemplatesResult::default()) } } @@ -467,7 +504,10 @@ pub async fn extract_and_store_workspace_id( /// Setup the MCP server with HTTP transport pub async fn setup_mcp_server() -> anyhow::Result<(Router, Arc)> { let session_manager = Arc::new(LocalSessionManager::default()); - let service_config = Default::default(); + let service_config = StreamableHttpServerConfig { + sse_keep_alive: Some(Duration::from_secs(15)), + stateful_mode: false, + }; let service = StreamableHttpService::new( || Ok(Runner::new()), session_manager.clone(), @@ -513,6 +553,5 @@ async fn list_mcp_tools_handler() -> JsonResult> { /// Creates a router service for listing MCP tools pub fn list_tools_service() -> Router { - Router::new() - .route("/", get(list_mcp_tools_handler)) -} \ No newline at end of file + Router::new().route("/", get(list_mcp_tools_handler)) +} diff --git a/backend/windmill-api/src/mcp/tools/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/tools/auto_generated_endpoints.rs index f2874fff47..629d4d6b8c 100644 --- a/backend/windmill-api/src/mcp/tools/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/tools/auto_generated_endpoints.rs @@ -1015,6 +1015,10 @@ You should get the schema of the script or flow before creating the schedule to "cron_version": { "type": "string", "description": "The version of the cron schedule to use (last is v2)" + }, + "dynamic_skip": { + "type": "string", + "description": "Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean." } }, "required": [ @@ -1135,6 +1139,10 @@ You should get the schema of the script or flow before updating the schedule to "cron_version": { "type": "string", "description": "The version of the cron schedule to use (last is v2)" + }, + "dynamic_skip": { + "type": "string", + "description": "Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean." } }, "required": [ diff --git a/backend/windmill-api/src/mcp/tools/endpoint_tools.rs b/backend/windmill-api/src/mcp/tools/endpoint_tools.rs index 1b8a7f0a66..882d9704f0 100644 --- a/backend/windmill-api/src/mcp/tools/endpoint_tools.rs +++ b/backend/windmill-api/src/mcp/tools/endpoint_tools.rs @@ -3,16 +3,15 @@ //! Contains the auto-generated endpoint tools and utilities for converting //! them to MCP tools and handling HTTP calls to Windmill API endpoints. -use rmcp::{model::Tool, Error}; -use std::sync::Arc; -use windmill_common::auth::create_jwt_token; -use windmill_common::db::Authed; -use windmill_common::BASE_URL; use crate::db::ApiAuthed; +use rmcp::{model::Tool, ErrorData}; +use std::sync::Arc; +use windmill_common::db::Authed; +use windmill_common::{auth::create_jwt_token, BASE_INTERNAL_URL}; // Import the auto-generated tools use super::auto_generated_endpoints; -pub use auto_generated_endpoints::{EndpointTool, all_tools}; +pub use auto_generated_endpoints::{all_tools, EndpointTool}; /// Get all available endpoint tools pub fn all_endpoint_tools() -> Vec { @@ -21,25 +20,28 @@ pub fn all_endpoint_tools() -> Vec { /// Convert endpoint tools to MCP tools pub fn endpoint_tools_to_mcp_tools(endpoint_tools: Vec) -> Vec { - endpoint_tools.into_iter().map(|tool| endpoint_tool_to_mcp_tool(&tool)).collect() + endpoint_tools + .into_iter() + .map(|tool| endpoint_tool_to_mcp_tool(&tool)) + .collect() } /// Convert a single endpoint tool to MCP tool pub fn endpoint_tool_to_mcp_tool(tool: &EndpointTool) -> Tool { let mut combined_properties = serde_json::Map::new(); let mut combined_required = Vec::new(); - + // Combine all parameter schemas let schemas = [ &tool.path_params_schema, - &tool.query_params_schema, + &tool.query_params_schema, &tool.body_schema, ]; - + for schema in schemas.iter().filter_map(|s| s.as_ref()) { merge_schema_into(&mut combined_properties, &mut combined_required, schema); } - + let combined_schema = serde_json::json!({ "type": "object", "properties": combined_properties, @@ -47,7 +49,7 @@ pub fn endpoint_tool_to_mcp_tool(tool: &EndpointTool) -> Tool { }); let description = format!("{}. {}", tool.description, tool.instructions); - + // Create annotations based on HTTP method and endpoint characteristics let annotations = create_endpoint_annotations(tool); @@ -55,6 +57,9 @@ pub fn endpoint_tool_to_mcp_tool(tool: &EndpointTool) -> Tool { name: tool.name.clone(), description: Some(description.into()), input_schema: Arc::new(combined_schema.as_object().unwrap().clone()), + title: Some(tool.name.to_string()), + output_schema: None, + icons: None, annotations: Some(annotations), } } @@ -62,15 +67,15 @@ pub fn endpoint_tool_to_mcp_tool(tool: &EndpointTool) -> Tool { /// Create appropriate annotations for endpoint tools based on HTTP method fn create_endpoint_annotations(tool: &EndpointTool) -> rmcp::model::ToolAnnotations { let method = tool.method.as_ref(); - + // Determine characteristics based on HTTP method let (read_only, destructive, idempotent, open_world) = match method { - "GET" => (true, false, true, true), // Read-only, safe, idempotent - "POST" => (false, true, false, true), // Can modify, potentially destructive, not idempotent - "PUT" => (false, false, true, true), // Can modify, typically idempotent updates - "DELETE" => (false, true, true, true), // Destructive but idempotent + "GET" => (true, false, true, true), // Read-only, safe, idempotent + "POST" => (false, true, false, true), // Can modify, potentially destructive, not idempotent + "PUT" => (false, false, true, true), // Can modify, typically idempotent updates + "DELETE" => (false, true, true, true), // Destructive but idempotent "PATCH" => (false, false, false, true), // Partial updates, not guaranteed idempotent - _ => (false, true, false, true), // Default: assume can modify and be destructive + _ => (false, true, false, true), // Default: assume can modify and be destructive }; rmcp::model::ToolAnnotations { @@ -93,7 +98,7 @@ fn merge_schema_into( combined_properties.insert(key.clone(), value.clone()); } } - + if let Some(required) = schema.get("required").and_then(|r| r.as_array()) { for req in required.iter().filter_map(|r| r.as_str()) { combined_required.push(req.to_string()); @@ -107,34 +112,52 @@ pub async fn call_endpoint_tool( args: serde_json::Value, workspace_id: &str, api_authed: &ApiAuthed, -) -> Result { +) -> Result { let args_map = match &args { serde_json::Value::Object(map) => map, - _ => return Err(Error::invalid_params("Arguments must be an object", Some(tool.name.clone().into()))), + _ => { + return Err(ErrorData::invalid_params( + "Arguments must be an object", + Some(tool.name.clone().into()), + )) + } }; // Build URL with path substitutions - let path_template = substitute_path_params(&tool.path, workspace_id, args_map, &tool.path_params_schema)?; + let path_template = + substitute_path_params(&tool.path, workspace_id, args_map, &tool.path_params_schema)?; let query_string = build_query_string(args_map, &tool.query_params_schema); - let full_url = format!("{}/api{}{}", BASE_URL.read().await, path_template, query_string); + let full_url = format!( + "{}/api{}{}", + BASE_INTERNAL_URL.as_str(), + path_template, + query_string + ); // Prepare request body let body_json = build_request_body(&tool.method, args_map, &tool.body_schema); // Create and execute request - let response = create_http_request(&tool.method, &full_url, workspace_id, api_authed, body_json).await?; - + let response = + create_http_request(&tool.method, &full_url, workspace_id, api_authed, body_json).await?; + let status = response.status(); let response_text = response.text().await.map_err(|e| { - Error::internal_error(format!("Failed to read response text: {}", e), None) + ErrorData::internal_error(format!("Failed to read response text: {}", e), None) })?; if status.is_success() { - Ok(serde_json::from_str(&response_text).unwrap_or_else(|_| serde_json::Value::String(response_text))) + Ok(serde_json::from_str(&response_text) + .unwrap_or_else(|_| serde_json::Value::String(response_text))) } else { - Err(Error::internal_error( - format!("HTTP {} {}: {}", status.as_u16(), status.canonical_reason().unwrap_or(""), response_text), - None + Err(ErrorData::internal_error( + format!( + "HTTP {} {}: {}", + status.as_u16(), + status.canonical_reason().unwrap_or(""), + response_text + ), + None, )) } } @@ -145,9 +168,9 @@ fn substitute_path_params( workspace_id: &str, args_map: &serde_json::Map, path_schema: &Option, -) -> Result { +) -> Result { let mut path_template = path.replace("{workspace}", workspace_id); - + if let Some(schema) = path_schema { if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) { for (param_name, _) in props { @@ -157,19 +180,19 @@ fn substitute_path_params( if let Some(str_val) = param_value.as_str() { path_template = path_template.replace(&placeholder, str_val); } - }, + } None => { tracing::warn!("Missing required path parameter: {}", param_name); - return Err(Error::invalid_params( + return Err(ErrorData::invalid_params( format!("Missing required path parameter: {}", param_name), - None + None, )); } } } } } - + Ok(path_template) } @@ -178,25 +201,31 @@ fn build_query_string( args_map: &serde_json::Map, query_schema: &Option, ) -> String { - let Some(schema) = query_schema else { return String::new() }; - let Some(props) = schema.get("properties").and_then(|p| p.as_object()) else { return String::new() }; - + let Some(schema) = query_schema else { + return String::new(); + }; + let Some(props) = schema.get("properties").and_then(|p| p.as_object()) else { + return String::new(); + }; + let query_params: Vec = props .keys() .filter_map(|param_name| { - args_map.get(param_name) + args_map + .get(param_name) .filter(|v| !v.is_null()) .map(|value| { let value_str = value.to_string(); let str_val = value_str.trim_matches('"'); - format!("{}={}", - urlencoding::encode(param_name), + format!( + "{}={}", + urlencoding::encode(param_name), urlencoding::encode(str_val) ) }) }) .collect(); - + if query_params.is_empty() { String::new() } else { @@ -213,18 +242,19 @@ fn build_request_body( if method == "GET" { return None; } - + let schema = body_schema.as_ref()?; let props = schema.get("properties")?.as_object()?; - + let body_map: serde_json::Map = props .keys() .filter_map(|param_name| { - args_map.get(param_name) + args_map + .get(param_name) .map(|value| (param_name.clone(), value.clone())) }) .collect(); - + if body_map.is_empty() { None } else { @@ -239,7 +269,7 @@ async fn create_http_request( workspace_id: &str, api_authed: &ApiAuthed, body_json: Option, -) -> Result { +) -> Result { let client = &crate::HTTP_CLIENT; let mut request_builder = match method { "GET" => client.get(url), @@ -247,16 +277,19 @@ async fn create_http_request( "PUT" => client.put(url), "DELETE" => client.delete(url), "PATCH" => client.patch(url), - _ => return Err(Error::invalid_params( - format!("Unsupported HTTP method: {}", method), - None - )), + _ => { + return Err(ErrorData::invalid_params( + format!("Unsupported HTTP method: {}", method), + None, + )) + } }; // Add authorization header let authed = Authed::from(api_authed.clone()); - let token = create_jwt_token(authed, workspace_id, 3600, None, None, None, None).await - .map_err(|e| Error::internal_error(e.to_string(), None))?; + let token = create_jwt_token(authed, workspace_id, 3600, None, None, None, None) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; request_builder = request_builder.header("Authorization", format!("Bearer {}", token)); // Add body if present @@ -266,7 +299,8 @@ async fn create_http_request( .json(&body); } - request_builder.send().await.map_err(|e| { - Error::internal_error(format!("Failed to execute request: {}", e), None) - }) -} \ No newline at end of file + request_builder + .send() + .await + .map_err(|e| ErrorData::internal_error(format!("Failed to execute request: {}", e), None)) +} diff --git a/backend/windmill-api/src/mcp/utils/database.rs b/backend/windmill-api/src/mcp/utils/database.rs index 0a0ea15ca6..39aac1ae6e 100644 --- a/backend/windmill-api/src/mcp/utils/database.rs +++ b/backend/windmill-api/src/mcp/utils/database.rs @@ -3,28 +3,32 @@ //! Contains all database query functions and database-related utilities //! used by the MCP server implementation. -use rmcp::Error; +use rmcp::ErrorData; use sql_builder::prelude::*; use windmill_common::db::UserDB; use windmill_common::scripts::{get_full_hub_script_by_path, Schema}; use windmill_common::utils::{query_elems_from_hub, StripPath}; use windmill_common::{DB, HUB_BASE_URL}; +use super::models::*; use crate::db::ApiAuthed; use crate::HTTP_CLIENT; -use super::models::*; /// Check if the user has proper MCP scopes -pub fn check_scopes(authed: &ApiAuthed) -> Result<(), Error> { +pub fn check_scopes(authed: &ApiAuthed) -> Result<(), ErrorData> { let scopes = authed.scopes.as_ref(); if scopes.is_none() - || scopes - .unwrap() - .iter() - .all(|scope| !scope.starts_with("mcp:all") && !scope.starts_with("mcp:favorites") && !scope.starts_with("mcp:hub:")) + || scopes.unwrap().iter().all(|scope| { + !scope.starts_with("mcp:all") + && !scope.starts_with("mcp:favorites") + && !scope.starts_with("mcp:hub:") + }) { tracing::error!("Unauthorized: missing mcp scope"); - return Err(Error::internal_error("Unauthorized: missing mcp scope".to_string(), None)); + return Err(ErrorData::internal_error( + "Unauthorized: missing mcp scope".to_string(), + None, + )); } Ok(()) } @@ -36,7 +40,7 @@ pub async fn get_item_schema( authed: &ApiAuthed, workspace_id: &str, item_type: &str, -) -> Result, Error> { +) -> Result, ErrorData> { let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); sqlb.fields(&["o.schema"]); sqlb.and_where("o.path = ?".bind(&path)); @@ -45,23 +49,23 @@ pub async fn get_item_schema( sqlb.and_where("o.draft_only IS NOT TRUE"); let sql = sqlb.sql().map_err(|_e| { tracing::error!("failed to build sql: {}", _e); - Error::internal_error("failed to build sql", None) + ErrorData::internal_error("failed to build sql", None) })?; let mut tx = user_db .clone() .begin(authed) .await - .map_err(|_e| Error::internal_error("failed to begin transaction", None))?; + .map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?; let item = sqlx::query_as::<_, ItemSchema>(&sql) .fetch_one(&mut *tx) .await .map_err(|_e| { tracing::error!("failed to fetch item schema: {}", _e); - Error::internal_error("failed to fetch item schema", None) + ErrorData::internal_error("failed to fetch item schema", None) })?; tx.commit() .await - .map_err(|_e| Error::internal_error("failed to commit transaction", None))?; + .map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?; Ok(item.schema) } @@ -70,29 +74,29 @@ pub async fn get_resources_types( user_db: &UserDB, authed: &ApiAuthed, workspace_id: &str, -) -> Result, Error> { +) -> Result, ErrorData> { let mut sqlb = SqlBuilder::select_from("resource_type as o"); sqlb.fields(&["o.name", "o.description"]); sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); let sql = sqlb.sql().map_err(|_e| { tracing::error!("failed to build sql: {}", _e); - Error::internal_error("failed to build sql", None) + ErrorData::internal_error("failed to build sql", None) })?; let mut tx = user_db .clone() .begin(authed) .await - .map_err(|_e| Error::internal_error("failed to begin transaction", None))?; + .map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?; let rows = sqlx::query_as::<_, ResourceType>(&sql) .fetch_all(&mut *tx) .await .map_err(|_e| { tracing::error!("Failed to fetch resource types: {}", _e); - Error::internal_error("failed to fetch resource types", None) + ErrorData::internal_error("failed to fetch resource types", None) })?; tx.commit() .await - .map_err(|_e| Error::internal_error("failed to commit transaction", None))?; + .map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?; Ok(rows) } @@ -102,30 +106,30 @@ pub async fn get_resources( authed: &ApiAuthed, workspace_id: &str, resource_type: &str, -) -> Result, Error> { +) -> Result, ErrorData> { let mut sqlb = SqlBuilder::select_from("resource as o"); sqlb.fields(&["o.path", "o.description", "o.resource_type"]); sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); sqlb.and_where("o.resource_type = ?".bind(&resource_type)); let sql = sqlb.sql().map_err(|_e| { tracing::error!("failed to build sql: {}", _e); - Error::internal_error("failed to build sql", None) + ErrorData::internal_error("failed to build sql", None) })?; let mut tx = user_db .clone() .begin(authed) .await - .map_err(|_e| Error::internal_error("failed to begin transaction", None))?; + .map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?; let rows = sqlx::query_as::<_, ResourceInfo>(&sql) .fetch_all(&mut *tx) .await .map_err(|_e| { tracing::error!("Failed to fetch resources: {}", _e); - Error::internal_error("failed to fetch resources", None) + ErrorData::internal_error("failed to fetch resources", None) })?; tx.commit() .await - .map_err(|_e| Error::internal_error("failed to commit transaction", None))?; + .map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?; Ok(rows) } @@ -138,7 +142,7 @@ pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen scope_type: &str, item_type: &str, scope_path: Option<&str>, -) -> Result, Error> { +) -> Result, ErrorData> { let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); let fields = vec!["o.path", "o.summary", "o.description", "o.schema"]; sqlb.fields(&fields); @@ -157,9 +161,15 @@ pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen // scope path is always a folder path, format is f/my_folder/* if let Some(scope_path) = scope_path { - if scope_path.split("/").count() != 3 || !scope_path.starts_with("f/") || !scope_path.ends_with("/*") { - return Err(Error::internal_error( - format!("Invalid folder format: {}, expected format is f/my_folder/*", scope_path), + if scope_path.split("/").count() != 3 + || !scope_path.starts_with("f/") + || !scope_path.ends_with("/*") + { + return Err(ErrorData::internal_error( + format!( + "Invalid folder format: {}, expected format is f/my_folder/*", + scope_path + ), None, )); } @@ -177,23 +187,23 @@ pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen .limit(100); let sql = sqlb.sql().map_err(|_e| { tracing::error!("failed to build sql: {}", _e); - Error::internal_error("failed to build sql", None) + ErrorData::internal_error("failed to build sql", None) })?; let mut tx = user_db .clone() .begin(authed) .await - .map_err(|_e| Error::internal_error("failed to begin transaction", None))?; + .map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?; let rows = sqlx::query_as::<_, T>(&sql) .fetch_all(&mut *tx) .await .map_err(|_e| { tracing::error!("Failed to fetch {}: {}", item_type, _e); - Error::internal_error(format!("failed to fetch {}", item_type), None) + ErrorData::internal_error(format!("failed to fetch {}", item_type), None) })?; tx.commit() .await - .map_err(|_e| Error::internal_error("failed to commit transaction", None))?; + .map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?; Ok(rows) } @@ -201,7 +211,7 @@ pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen pub async fn get_scripts_from_hub( db: &DB, scope_integrations: Option<&str>, -) -> Result, Error> { +) -> Result, ErrorData> { let query_params = Some(vec![ ("limit", "100".to_string()), ("with_schema", "true".to_string()), @@ -213,34 +223,34 @@ pub async fn get_scripts_from_hub( .await .map_err(|e| { tracing::error!("Failed to get items from hub: {}", e); - Error::internal_error(format!("Failed to get items from hub: {}", e), None) + ErrorData::internal_error(format!("Failed to get items from hub: {}", e), None) })?; - + use axum::body::to_bytes; let body_bytes = to_bytes(response, usize::MAX).await.map_err(|e| { tracing::error!("Failed to read response body: {}", e); - Error::internal_error(format!("Failed to read response body: {}", e), None) + ErrorData::internal_error(format!("Failed to read response body: {}", e), None) })?; let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| { tracing::error!("Failed to decode response body: {}", e); - Error::internal_error(format!("Failed to decode response body: {}", e), None) + ErrorData::internal_error(format!("Failed to decode response body: {}", e), None) })?; let hub_response: HubResponse = serde_json::from_str(&body_str).map_err(|e| { tracing::error!("Failed to parse hub response: {}", e); - Error::internal_error(format!("Failed to parse hub response: {}", e), None) + ErrorData::internal_error(format!("Failed to parse hub response: {}", e), None) })?; Ok(hub_response.asks) } /// Get the schema for a Hub script -pub async fn get_hub_script_schema(path: &str, db: &DB) -> Result, Error> { +pub async fn get_hub_script_schema(path: &str, db: &DB) -> Result, ErrorData> { let strip_path = StripPath(path.to_string()); let res = get_full_hub_script_by_path(strip_path, &HTTP_CLIENT, Some(db)) .await .map_err(|e| { tracing::error!("Failed to get hub script: {}", e); - Error::internal_error(format!("Failed to get hub script: {}", e), None) + ErrorData::internal_error(format!("Failed to get hub script: {}", e), None) })?; match serde_json::from_str::(res.schema.get()) { Ok(schema) => Ok(Some(schema)), @@ -249,4 +259,4 @@ pub async fn get_hub_script_schema(path: &str, db: &DB) -> Result Ok(None) } } -} \ No newline at end of file +} diff --git a/backend/windmill-api/src/mcp/utils/schema.rs b/backend/windmill-api/src/mcp/utils/schema.rs index b11f2f83ab..0de27a677f 100644 --- a/backend/windmill-api/src/mcp/utils/schema.rs +++ b/backend/windmill-api/src/mcp/utils/schema.rs @@ -3,16 +3,16 @@ //! Contains functions for transforming Windmill schemas into MCP-compatible formats, //! including resource enrichment and schema conversion utilities. -use rmcp::Error; +use rmcp::ErrorData; use serde_json::Value; use std::collections::HashMap; use windmill_common::db::UserDB; use windmill_common::scripts::Schema; -use crate::db::ApiAuthed; -use super::models::{SchemaType, ResourceInfo, ResourceType}; use super::database::get_resources; +use super::models::{ResourceInfo, ResourceType, SchemaType}; use super::transform::apply_key_transformation; +use crate::db::ApiAuthed; /// Convert a Windmill Schema to a SchemaType pub fn convert_schema_to_schema_type(schema: Option) -> SchemaType { @@ -35,7 +35,7 @@ pub async fn transform_schema_for_resources( w_id: &str, resources_cache: &mut HashMap>, resources_types: &Vec, -) -> Result { +) -> Result { let mut schema_obj: SchemaType = schema.clone(); // replace invalid char in property key with underscore @@ -71,24 +71,15 @@ pub async fn transform_schema_for_resources( let resource_type_obj = resource_type.cloned(); if !resources_cache.contains_key(&resource_type_key) { - let available_resources = get_resources( - user_db, - authed, - &w_id, - &resource_type_key, - ) - .await; + let available_resources = + get_resources(user_db, authed, &w_id, &resource_type_key).await; match available_resources { Ok(cache_data) => { - resources_cache - .insert(resource_type_key.clone(), cache_data); + resources_cache.insert(resource_type_key.clone(), cache_data); } Err(e) => { - tracing::error!( - "Failed to fetch resource cache data: {}", - e - ); + tracing::error!("Failed to fetch resource cache data: {}", e); continue; // Skip this property if fetching failed } } @@ -111,24 +102,16 @@ pub async fn transform_schema_for_resources( ), None => "An object parameter.".to_string() }; - prop_map.insert( - "type".to_string(), - Value::String("string".to_string()), - ); - prop_map.insert( - "description".to_string(), - Value::String(description), - ); + prop_map + .insert("type".to_string(), Value::String("string".to_string())); + prop_map.insert("description".to_string(), Value::String(description)); if resources_count > 0 { let resources_description = resource_cache .iter() .map(|resource| { format!( "{}: $res:{}", - resource - .description - .as_deref() - .unwrap_or("No title"), + resource.description.as_deref().unwrap_or("No title"), resource.path ) }) @@ -157,4 +140,4 @@ pub async fn transform_schema_for_resources( } Ok(schema_obj) -} \ No newline at end of file +} diff --git a/backend/windmill-api/src/mcp/utils/transform.rs b/backend/windmill-api/src/mcp/utils/transform.rs index c40fe37263..d4ef52fd3f 100644 --- a/backend/windmill-api/src/mcp/utils/transform.rs +++ b/backend/windmill-api/src/mcp/utils/transform.rs @@ -5,22 +5,34 @@ use super::models::SchemaType; +// MCP clients do not allow names longer than 60 characters +const MAX_PATH_LENGTH: usize = 60; + /// Transform the path for workspace scripts/flows -/// -/// This function takes a path and a type string and formats the transformed -/// path with the type prefix. This is used when listing, because we can't -/// have names with slashes. Because we replace slashes with underscores, +/// +/// This function takes a path and a type string and formats the transformed +/// path with the type prefix. This is used when listing, because we can't +/// have names with slashes. Because we replace slashes with underscores, /// we also need to escape underscores. pub fn transform_path(path: &str, type_str: &str) -> String { let escaped_path = path.replace('_', "__").replace('/', "_"); // first letter of type_str is used as prefix, only one letter to avoid reaching 60 char name limit - format!("{}-{}", &type_str[..1], escaped_path) + let transformed_path = format!("{}-{}", &type_str[..1], escaped_path); + if transformed_path.len() > MAX_PATH_LENGTH { + let suffix = "_TRUNC"; + return format!( + "{}{}", + &transformed_path[..MAX_PATH_LENGTH - suffix.len()], + suffix + ); + } + transformed_path } /// Reverse the transformation of a path /// -/// This function takes a transformed path and reverses the transformation -/// applied by `transform_path`. It checks if the path starts with "h" +/// This function takes a transformed path and reverses the transformation +/// applied by `transform_path`. It checks if the path starts with "h" /// (indicating a Hub script) and removes the prefix if present. /// It then determines the type of the item (script or flow) based on the prefix. /// This is used in call_tool to get the original path, and the type of the item. @@ -54,7 +66,10 @@ pub fn reverse_transform(transformed_path: &str) -> Result<(&str, String, bool), parts[0].to_string() } else { const TEMP_PLACEHOLDER: &str = "@@UNDERSCORE@@"; - mangled_path.replace("__", TEMP_PLACEHOLDER).replace('_', "/").replace(TEMP_PLACEHOLDER, "_") + mangled_path + .replace("__", TEMP_PLACEHOLDER) + .replace('_', "/") + .replace(TEMP_PLACEHOLDER, "_") }; Ok((type_str, original_path, is_hub)) @@ -64,7 +79,7 @@ pub fn reverse_transform(transformed_path: &str) -> Result<(&str, String, bool), /// /// This function takes a key and replaces spaces with underscores. /// It also removes any characters that are not alphanumeric or underscores. -/// This is used when listing, because we can't have names with spaces +/// This is used when listing, because we can't have names with spaces /// or special characters in the schema properties. pub fn apply_key_transformation(key: &str) -> String { key.replace(' ', "_") @@ -75,8 +90,8 @@ pub fn apply_key_transformation(key: &str) -> String { /// Reverse the transformation of a key /// -/// This function takes a transformed key and a schema object and reverses -/// the transformation applied by `apply_key_transformation`. This can be +/// This function takes a transformed key and a schema object and reverses +/// the transformation applied by `apply_key_transformation`. This can be /// subject to collisions, but it's unlikely and is ok for our use case. pub fn reverse_transform_key(transformed_key: &str, schema_obj: &Option) -> String { let schema_obj = match schema_obj { @@ -98,4 +113,4 @@ pub fn reverse_transform_key(transformed_key: &str, schema_obj: &Option>, - db: &DB, - trigger: &MqttTrigger, -) -> anyhow::Result<()> { - let args = MqttTrigger::build_job_args( - &trigger.script_path, - trigger.is_flow, - &trigger.workspace_id, - db, - payload, - trigger_info, - ) - .await?; - - let authed = fetch_api_authed( - trigger.edited_by.clone(), - trigger.email.clone(), - &trigger.workspace_id, - db, - Some(format!("mqtt-{}", trigger.path)), - ) - .await?; - - trigger_runnable( - db, - None, - authed, - &trigger.workspace_id, - &trigger.script_path, - trigger.is_flow, - args, - trigger.retry.as_ref(), - trigger.error_handler_path.as_deref(), - trigger.error_handler_args.as_ref(), - format!("mqtt_trigger/{}", trigger.path), - ) - .await?; - - Ok(()) -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct MqttV3Config { - clean_session: Option, -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct MqttV5Config { - clean_start: Option, - session_expiry_interval: Option, - topic_alias_maximum: Option, -} - -#[derive(Clone, Debug, Deserialize, Serialize, Type)] -#[serde(rename_all = "lowercase")] -pub enum QualityOfService { - Qos0, - Qos1, - Qos2, -} - -impl From for V3QoS { - fn from(value: QualityOfService) -> Self { - match value { - QualityOfService::Qos0 => V3QoS::AtMostOnce, - QualityOfService::Qos1 => V3QoS::AtLeastOnce, - QualityOfService::Qos2 => V3QoS::ExactlyOnce, - } - } -} - -impl From for V5QoS { - fn from(value: QualityOfService) -> Self { - match value { - QualityOfService::Qos0 => V5QoS::AtMostOnce, - QualityOfService::Qos1 => V5QoS::AtLeastOnce, - QualityOfService::Qos2 => V5QoS::ExactlyOnce, - } - } -} - -#[derive(Debug, Deserialize, Serialize, Type)] -#[sqlx(type_name = "MQTT_CLIENT_VERSION")] -#[sqlx(rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum MqttClientVersion { - V3, - V5, -} - -#[derive(Debug, Deserialize)] -pub struct Tls { - enabled: bool, - ca_certificate: String, - //encoded in base64 - pkcs12_client_certificate: Option, - pkcs12_certificate_password: Option, -} - -#[derive(Debug, Deserialize)] -pub struct Credentials { - username: Option, - password: Option, -} - -#[derive(Debug, Deserialize)] -pub struct MqttResource { - broker: String, - port: u16, - credentials: Option, - tls: Option, -} -#[derive(Clone, Debug, FromRow, Serialize, Deserialize)] -pub struct SubscribeTopic { - qos: QualityOfService, - topic: String, -} - -struct MqttClientBuilder<'client> { - mqtt_resource: MqttResource, - client_id: &'client str, - subscribe_topics: Vec, - v3_config: Option<&'client MqttV3Config>, - v5_config: Option<&'client MqttV5Config>, - mqtt_client_version: Option<&'client MqttClientVersion>, -} - -impl<'client> MqttClientBuilder<'client> { - fn new( - mqtt_resource: MqttResource, - client_id: Option<&'client str>, - subscribe_topics: Vec, - v3_config: Option<&'client MqttV3Config>, - v5_config: Option<&'client MqttV5Config>, - mqtt_client_version: Option<&'client MqttClientVersion>, - ) -> Self { - Self { - mqtt_resource, - client_id: client_id.unwrap_or(""), - subscribe_topics, - v3_config, - v5_config, - mqtt_client_version, - } - } - - async fn build_client(&self) -> Result { - match self.mqtt_client_version { - Some(MqttClientVersion::V5) | None => self.build_v5_client().await, - Some(MqttClientVersion::V3) => self.build_v3_client().await, - } - } - - fn get_tls_configuration(&self) -> Result> { - let transport = match self.mqtt_resource.tls { - Some(ref tls) if tls.enabled => { - let transport = match tls.ca_certificate.trim().is_empty() { - true => rumqttc::Transport::Tls(TlsConfiguration::Native), - false => rumqttc::Transport::Tls(TlsConfiguration::SimpleNative { - ca: tls.ca_certificate.as_bytes().to_vec(), - client_auth: { - match tls.pkcs12_client_certificate.as_ref() { - Some(client_certificate) - if !client_certificate.trim().is_empty() => - { - let client_certificate = BASE64_STANDARD - .decode(client_certificate) - .map_err(to_anyhow)?; - let password = tls - .pkcs12_certificate_password - .clone() - .unwrap_or("".to_string()); - Some((client_certificate, password)) - } - _ => None, - } - }, - }), - }; - - Some(transport) - } - _ => None, - }; - - Ok(transport) - } - - async fn build_v5_client(&self) -> Result { - let mut mqtt_options = V5MqttOptions::new( - self.client_id, - &self.mqtt_resource.broker, - self.mqtt_resource.port, - ); - - if let Some(credentials) = &self.mqtt_resource.credentials { - let username = credentials.username.as_deref().unwrap_or(""); - let password = credentials.password.as_deref().unwrap_or(""); - mqtt_options.set_credentials(username, password); - } - - if let Some(transport) = self.get_tls_configuration()? { - mqtt_options.set_transport(transport); - } - - mqtt_options.set_connection_timeout(CLIENT_CONNECTION_TIMEOUT); - - mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); - - if let Some(v5_config) = self.v5_config { - mqtt_options.set_clean_start(v5_config.clean_start.unwrap_or(true)); - mqtt_options.set_connect_properties(ConnectProperties { - session_expiry_interval: v5_config.session_expiry_interval, - receive_maximum: None, - max_packet_size: None, - topic_alias_max: v5_config.topic_alias_maximum.or(Some(TOPIC_ALIAS_MAXIMUM)), - request_response_info: None, - request_problem_info: None, - user_properties: vec![], - authentication_method: None, - authentication_data: None, - }); - } - - let (async_client, mut event_loop) = - V5AsyncClient::new(mqtt_options, self.subscribe_topics.len()); - event_loop.verify_connection().await?; - - if !self.subscribe_topics.is_empty() { - let subscribe_filters = self - .subscribe_topics - .iter() - .map(|topic| Filter::new(topic.topic.clone(), topic.qos.clone().into())) - .collect_vec(); - - async_client - .subscribe_many(subscribe_filters) - .await - .map_err(to_anyhow)?; - } - Ok(MqttClientResult::V5((V5MqttHandler, event_loop))) - } - - async fn build_v3_client(&self) -> Result { - let mut mqtt_options = V3MqttOptions::new( - self.client_id, - &self.mqtt_resource.broker, - self.mqtt_resource.port, - ); - - if let Some(credentials) = &self.mqtt_resource.credentials { - let username = credentials.username.as_deref().unwrap_or(""); - let password = credentials.password.as_deref().unwrap_or(""); - mqtt_options.set_credentials(username, password); - } - - if let Some(transport) = self.get_tls_configuration()? { - mqtt_options.set_transport(transport); - } - mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); - if let Some(v3_config) = self.v3_config { - mqtt_options.set_clean_session(v3_config.clean_session.unwrap_or(true)); - } - - let (async_client, mut event_loop) = - V3AsyncClient::new(mqtt_options, self.subscribe_topics.len()); - event_loop.verify_connection().await?; - - if !self.subscribe_topics.is_empty() { - let subscribe_filters = self - .subscribe_topics - .iter() - .map(|topic| SubscribeFilter::new(topic.topic.clone(), topic.qos.clone().into())) - .collect_vec(); - - async_client - .subscribe_many(subscribe_filters) - .await - .map_err(to_anyhow)?; - } - Ok(MqttClientResult::V3((V3MqttHandler, event_loop))) - } -} - -#[derive(Debug, Serialize, Deserialize, FromRow)] -pub struct MqttTrigger { - pub mqtt_resource_path: String, - pub subscribe_topics: Vec>, - pub v3_config: Option>, - pub v5_config: Option>, - pub client_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub client_version: Option, - pub path: String, - pub script_path: String, - pub is_flow: bool, - pub workspace_id: String, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - pub extra_perms: Option, - pub error: Option, - pub server_id: Option, - pub last_server_ping: Option>, - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, -} - -const KEEP_ALIVE: u64 = 60; -const CLIENT_CONNECTION_TIMEOUT: u64 = 60; -const TOPIC_ALIAS_MAXIMUM: u16 = 65535; - -fn convert_disconnect_packet_into_err(disconnect: rumqttc::v5::mqttbytes::v5::Disconnect) -> Error { - let err_message = disconnect - .properties - .map(|properties| properties.reason_string) - .flatten(); - let reason_code = disconnect.reason_code as u8; - anyhow::anyhow!( - "Disconnected by the broker, reason code: {}, {}", - reason_code, - err_message - .map(|err| format!("message: {}", err)) - .unwrap_or("".to_string()) - ) - .into() -} - -async fn loop_ping(db: &DB, mqtt: &MqttConfig, error: Option<&str>) { - loop { - if mqtt.update_ping(db, error).await.is_none() { - return; - } - - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; - } -} - -enum MqttClientResult { - V3((V3MqttHandler, V3EventLoop)), - V5((V5MqttHandler, V5EventLoop)), -} - -trait MqttEvent { - type IncomingPacket; - type PublishPacket; - type Event; - - fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData; - fn handle_event(&self, event: Self::Event) -> Result>; -} - -struct V5MqttHandler; - -impl MqttEvent for V5MqttHandler { - type IncomingPacket = V5Incoming; - type PublishPacket = rumqttc::v5::mqttbytes::v5::Publish; - type Event = V5Event; - - fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData { - PublishData::new( - String::from_utf8(publish_packet.topic.as_ref().to_vec()).unwrap_or("".to_string()), - publish_packet.retain, - publish_packet.pkid, - publish_packet.properties, - publish_packet.qos as u8, - ) - } - - fn handle_event(&self, event: Self::Event) -> Result> { - tracing::debug!("Inside V5 event"); - match event { - Self::Event::Incoming(packet) => match packet { - Self::IncomingPacket::Publish(publish_packet) => { - return Ok(Some(( - publish_packet.payload.clone(), - Self::handle_publish_packet(publish_packet), - ))) - } - Self::IncomingPacket::Disconnect(disconnect) => { - return Err(convert_disconnect_packet_into_err(disconnect)); - } - packet => { - tracing::debug!("Received = {:#?}", packet); - } - }, - Self::Event::Outgoing(packet) => { - tracing::debug!("Outgoing Received = {:#?}", packet); - } - } - - Ok(None) - } -} - -struct V3MqttHandler; - -impl MqttEvent for V3MqttHandler { - type IncomingPacket = V3Incoming; - type PublishPacket = rumqttc::mqttbytes::v4::Publish; - type Event = V3Event; - - fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData { - PublishData::new( - publish_packet.topic, - publish_packet.retain, - publish_packet.pkid, - None, - publish_packet.qos as u8, - ) - } - - fn handle_event(&self, event: Self::Event) -> Result> { - tracing::debug!("Inside V3 event"); - match event { - Self::Event::Incoming(packet) => match packet { - Self::IncomingPacket::Publish(publish_packet) => { - return Ok(Some(( - publish_packet.payload.clone(), - Self::handle_publish_packet(publish_packet), - ))) - } - packet => { - tracing::debug!("Received = {:?}", packet); - } - }, - Self::Event::Outgoing(packet) => { - tracing::debug!("Outgoing Received = {:?}", packet); - } - } - - Ok(None) - } -} - -const TIMEOUT_DURATION: u64 = 10; -const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION); - -#[async_trait] -pub trait EventLoop { - type Event; - type Error; - - async fn poll(&mut self) -> Result; - async fn verify_connection(&mut self) -> Result<()>; -} - -#[async_trait] -impl EventLoop for V5EventLoop { - type Event = V5Event; - type Error = rumqttc::v5::ConnectionError; - async fn poll(&mut self) -> Result { - self.poll().await.map_err(|err| to_anyhow(err).into()) - } - - async fn verify_connection(&mut self) -> Result<()> { - let start = std::time::Instant::now(); - - while start.elapsed() < CONNECTION_TIMEOUT { - match self.poll().await.map_err(to_anyhow)? { - Self::Event::Incoming(V5Incoming::ConnAck(_)) => return Ok(()), - Self::Event::Incoming(V5Incoming::Disconnect(disconnect)) => { - return Err(convert_disconnect_packet_into_err(disconnect)); - } - _ => continue, - } - } - - Err(Error::BadConfig(format!( - "Timeout occurred while trying to connect to mqtt broker after {} seconds", - TIMEOUT_DURATION - ))) - } -} - -#[async_trait] -impl EventLoop for V3EventLoop { - type Event = V3Event; - type Error = Error; - - async fn poll(&mut self) -> Result { - self.poll().await.map_err(|err| to_anyhow(err).into()) - } - - async fn verify_connection(&mut self) -> Result<()> { - let start = std::time::Instant::now(); - - while start.elapsed() < CONNECTION_TIMEOUT { - match self.poll().await.map_err(to_anyhow)? { - Self::Event::Incoming(rumqttc::Packet::ConnAck(_)) => return Ok(()), - _ => continue, - } - } - - Err(Error::BadConfig(format!( - "Timeout occurred while trying to connect to mqtt broker after {} seconds", - TIMEOUT_DURATION - ))) - } -} - -async fn handle_publish_packet(db: &DB, mqtt: &MqttConfig, payload: Bytes, publish: PublishData) { - let trigger_info = HashMap::from([ - ("topic".to_string(), to_raw_value(&publish.topic)), - ("retain".to_string(), to_raw_value(&publish.retain)), - ("pkid".to_string(), to_raw_value(&publish.pkid)), - ("qos".to_string(), to_raw_value(&publish.qos)), - ( - "v5".to_string(), - to_raw_value(&publish.v5.map(|properties| { - serde_json::json!({ - "payload_format_indicator": properties.payload_format_indicator, - "topic_alias": properties.topic_alias, - "response_topic": properties.response_topic, - "correlation_data": properties.correlation_data.as_deref(), - "user_properties": properties.user_properties, - "subscription_identifiers": properties.subscription_identifiers, - "content_type": properties.content_type, - }) - })), - ), - ]); - mqtt.handle(&db, payload.as_ref(), trigger_info).await; -} - -async fn handle_event(db: &DB, mqtt: &MqttConfig, handler: H, mut event_loop: E) -> () -where - H: MqttEvent, - E: EventLoop, - E::Error: ToString, -{ - loop { - let event = event_loop.poll().await; - - match event { - Ok(event) => { - let publish_data = handler.handle_event(event); - if let Ok(Some((payload, publish_data))) = publish_data { - handle_publish_packet(db, mqtt, payload, publish_data).await; - } - } - Err(err) => { - let err = err.to_string(); - tracing::debug!("Error: {}", &err); - mqtt.disable_with_error(&db, err).await; - return; - } - } - } -} - -#[derive(Debug)] -enum MqttConfig { - Trigger(MqttTrigger), - Capture(CaptureConfigForMqttTrigger), -} - -impl MqttConfig { - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - match self { - MqttConfig::Trigger(trigger) => trigger.update_ping(db, error).await, - MqttConfig::Capture(capture) => capture.update_ping(db, error).await, - } - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - match self { - MqttConfig::Trigger(trigger) => trigger.disable_with_error(&db, error).await, - MqttConfig::Capture(capture) => capture.disable_with_error(db, error).await, - } - } - - async fn start_consuming_messages( - &self, - db: &DB, - ) -> std::result::Result { - let mqtt_resource_path; - let subscribe_topics; - let workspace_id; - let authed; - let client_version; - let client_id; - let v3_config; - let v5_config; - match self { - MqttConfig::Capture(capture) => { - mqtt_resource_path = &capture.trigger_config.0.mqtt_resource_path; - subscribe_topics = capture.trigger_config.0.subscribe_topics.clone(); - workspace_id = &capture.workspace_id; - authed = capture.fetch_authed(&db).await?; - client_version = capture.trigger_config.0.client_version.as_ref(); - client_id = capture.trigger_config.0.client_id.as_deref(); - v3_config = capture.trigger_config.0.v3_config.as_ref(); - v5_config = capture.trigger_config.0.v5_config.as_ref(); - } - MqttConfig::Trigger(trigger) => { - mqtt_resource_path = &trigger.mqtt_resource_path; - subscribe_topics = trigger - .subscribe_topics - .iter() - .map(|topic| topic.0.clone()) - .collect_vec(); - workspace_id = &trigger.workspace_id; - client_version = trigger.client_version.as_ref(); - authed = trigger.fetch_authed(&db).await?; - client_id = trigger.client_id.as_deref(); - v3_config = trigger.v3_config.as_ref().map(|v3_config| &v3_config.0); - v5_config = trigger.v5_config.as_ref().map(|v5_config| &v5_config.0); - } - } - let mqtt_resource = try_get_resource_from_db_as::( - &authed, - Some(UserDB::new(db.clone())), - db, - mqtt_resource_path, - workspace_id, - ) - .await?; - let client_builder = MqttClientBuilder::new( - mqtt_resource, - client_id, - subscribe_topics, - v3_config, - v5_config, - client_version, - ); - - client_builder.build_client().await - } - - async fn handle( - &self, - db: &DB, - payload: &[u8], - trigger_info: HashMap>, - ) -> () { - match self { - MqttConfig::Trigger(trigger) => trigger.handle(&db, payload, trigger_info).await, - MqttConfig::Capture(capture) => capture.handle(&db, payload, trigger_info).await, - } - } -} - -impl MqttTrigger { - async fn try_to_listen_to_mqtt_messages( - self, - db: DB, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> () { - let mqtt_trigger = sqlx::query_scalar!( - r#" - UPDATE - mqtt_trigger - SET - server_id = $1, - last_server_ping = now(), - error = 'Connecting...' - WHERE - enabled IS TRUE - AND workspace_id = $2 - AND path = $3 - AND (last_server_ping IS NULL - OR last_server_ping < now() - INTERVAL '15 seconds' - ) - RETURNING true - "#, - *INSTANCE_NAME, - self.workspace_id, - self.path, - ) - .fetch_optional(&db) - .await; - match mqtt_trigger { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tracing::info!("Spawning new task to listen to mqtt notifications"); - tokio::spawn(async move { - listen_to_messages(MqttConfig::Trigger(self), db.clone(), killpill_rx) - .await; - }); - } else { - tracing::info!("Mqtt trigger {} already being listened to", self.path); - } - } - Err(err) => { - tracing::error!( - "Error acquiring lock for mqtt trigger {}: {:?}", - self.path, - err - ); - } - }; - } - - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - let updated = sqlx::query_scalar!( - r#" - UPDATE - mqtt_trigger - SET - last_server_ping = now(), - error = $1 - WHERE - workspace_id = $2 - AND path = $3 - AND server_id = $4 - AND enabled IS TRUE - RETURNING 1 - "#, - error, - &self.workspace_id, - &self.path, - *INSTANCE_NAME - ) - .fetch_optional(db) - .await; - - match updated { - Ok(updated) => { - if updated.flatten().is_none() { - // allow faster restart of mqtt trigger - sqlx::query!( - r#" - UPDATE - mqtt_trigger - SET - last_server_ping = NULL - WHERE - workspace_id = $1 - AND path = $2 - AND server_id IS NULL"#, - &self.workspace_id, - &self.path, - ) - .execute(db) - .await - .ok(); - tracing::info!( - "Mqtt trigger {} changed, disabled, or deleted, stopping...", - self.path - ); - return None; - } - } - Err(err) => { - tracing::warn!( - "Error updating ping of mqtt trigger {}: {:?}", - self.path, - err - ); - } - }; - - Some(()) - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - match sqlx::query!( - r#" - UPDATE - mqtt_trigger - SET - enabled = FALSE, - error = $1, - server_id = NULL, - last_server_ping = NULL - WHERE - workspace_id = $2 AND - path = $3 - "#, - error, - self.workspace_id, - self.path, - ) - .execute(db) - .await - { - Ok(_) => { - report_critical_error( - format!( - "Disabling mqtt trigger {} because of error: {}", - self.path, error - ), - db.clone(), - Some(&self.workspace_id), - None, - ) - .await; - } - Err(disable_err) => { - report_critical_error( - format!("Could not disable mqtt trigger {} with err {}, disabling because of error {}", self.path, disable_err, error), - db.clone(), - Some(&self.workspace_id), - None, - ).await; - } - } - } - - async fn fetch_authed(&self, db: &DB) -> Result { - fetch_api_authed( - self.edited_by.clone(), - self.email.clone(), - &self.workspace_id, - db, - Some(format!("mqtt-{}", self.path)), - ) - .await - } - - async fn handle( - &self, - db: &DB, - payload: &[u8], - trigger_info: HashMap>, - ) -> () { - if let Err(err) = run_job(payload, trigger_info, db, self).await { - report_critical_error( - format!("Failed to trigger job from mqtt {}: {:?}", self.path, err), - db.clone(), - Some(&self.workspace_id), - None, - ) - .await; - }; - } -} - -impl TriggerJobArgs<&[u8]> for MqttTrigger { - fn v1_payload_fn(payload: &[u8]) -> HashMap> { - HashMap::from([("payload".to_string(), to_raw_value(&payload))]) - } - - fn v2_payload_fn(payload: &[u8]) -> HashMap> { - let base64_payload = engine::general_purpose::STANDARD.encode(payload); - HashMap::from([("payload".to_string(), to_raw_value(&base64_payload))]) - } - - fn trigger_kind() -> TriggerKind { - TriggerKind::Mqtt - } -} - -struct PublishData { - topic: String, - retain: bool, - pkid: u16, - v5: Option, - qos: u8, -} - -impl PublishData { - fn new( - topic: String, - retain: bool, - pkid: u16, - v5: Option, - qos: u8, - ) -> PublishData { - PublishData { topic, retain, pkid, v5, qos } - } -} - -async fn listen_to_messages( - mqtt: MqttConfig, - db: DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) { - tokio::select! { - biased; - - _ = killpill_rx.recv() => { - return; - } - - _ = loop_ping(&db, &mqtt, Some("Connecting...")) => { - return; - } - - result = mqtt.start_consuming_messages(&db) => { - tokio::select! { - biased; - - _ = killpill_rx.recv() => { - return; - } - - _ = loop_ping(&db, &mqtt, None) => { - return; - } - - _ = async { - match result { - Ok(connection) => { - match connection { - MqttClientResult::V3((v3_handler, event_loop)) => handle_event(&db, &mqtt, v3_handler, event_loop).await, - MqttClientResult::V5((v5_handler, event_loop)) => handle_event(&db, &mqtt, v5_handler, event_loop).await, - } - } - Err(err) => { - tracing::error!( - "Mqtt trigger error while trying to start listening to notifications: {}", - &err - ); - mqtt.disable_with_error(&db, err.to_string()).await - } - } - } => {} - } - } - } -} - -#[derive(Debug, Deserialize)] -struct CaptureConfigForMqttTrigger { - trigger_config: SqlxJson, - path: String, - is_flow: bool, - workspace_id: String, - owner: String, - email: String, -} - -impl CaptureConfigForMqttTrigger { - async fn try_to_listen_to_mqtt_messages( - self, - db: DB, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> () { - match sqlx::query_scalar!( - r#" - UPDATE - capture_config - SET - server_id = $1, - last_server_ping = now(), - error = 'Connecting...' - WHERE - last_client_ping > NOW() - INTERVAL '10 seconds' AND - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = 'mqtt' AND - (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') - RETURNING true - "#, - *INSTANCE_NAME, - self.workspace_id, - self.path, - self.is_flow, - ) - .fetch_optional(&db) - .await - { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tokio::spawn(listen_to_messages( - MqttConfig::Capture(self), - db, - killpill_rx, - )); - } else { - tracing::info!("Mqtt {} already being listened to", self.path); - } - } - Err(err) => { - tracing::error!( - "Error acquiring lock for capture mqtt {}: {:?}", - self.path, - err - ); - } - }; - } - - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - match sqlx::query_scalar!( - r#" - UPDATE - capture_config - SET - last_server_ping = now(), - error = $1 - WHERE - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = 'mqtt' AND - server_id = $5 AND - last_client_ping > NOW() - INTERVAL '10 seconds' - RETURNING 1 - "#, - error, - self.workspace_id, - self.path, - self.is_flow, - *INSTANCE_NAME - ) - .fetch_optional(db) - .await - { - Ok(updated) => { - if updated.flatten().is_none() { - // allow faster restart of mqtt capture - sqlx::query!( - r#"UPDATE - capture_config - SET - last_server_ping = NULL - WHERE - workspace_id = $1 AND - path = $2 AND - is_flow = $3 AND - trigger_kind = 'mqtt' AND - server_id IS NULL - "#, - self.workspace_id, - self.path, - self.is_flow, - ) - .execute(db) - .await - .ok(); - tracing::info!( - "Mqtt capture {} changed, disabled, or deleted, stopping...", - self.path - ); - return None; - } - } - Err(err) => { - tracing::warn!( - "Error updating ping of capture mqtt {}: {:?}", - self.path, - err - ); - } - }; - - Some(()) - } - - async fn fetch_authed(&self, db: &DB) -> Result { - fetch_api_authed( - self.owner.clone(), - self.email.clone(), - &self.workspace_id, - db, - Some(format!("mqtt-{}", self.get_trigger_path())), - ) - .await - } - - fn get_trigger_path(&self) -> String { - format!( - "{}-{}", - if self.is_flow { "flow" } else { "script" }, - self.path - ) - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - if let Err(err) = sqlx::query!( - r#" - UPDATE - capture_config - SET - error = $1, - server_id = NULL, - last_server_ping = NULL - WHERE - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = 'mqtt' - "#, - error, - self.workspace_id, - self.path, - self.is_flow, - ) - .execute(db) - .await - { - tracing::error!( - "Could not disable mqtt capture {} ({}) with err {}, disabling because of error {}", - self.path, - self.workspace_id, - err, - error - ); - } - } - - async fn handle( - &self, - db: &DB, - payload: &[u8], - trigger_info: HashMap>, - ) -> () { - let (main_args, preprocessor_args) = - MqttTrigger::build_capture_payloads(payload, trigger_info); - if let Err(err) = insert_capture_payload( - db, - &self.workspace_id, - &self.path, - self.is_flow, - &TriggerKind::Mqtt, - main_args, - preprocessor_args, - &self.owner, - ) - .await - { - tracing::error!("Error inserting capture payload: {:?}", err); - } - } -} - -async fn listen_to_unlistened_mqtt_events( - db: &DB, - killpill_rx: &tokio::sync::broadcast::Receiver<()>, -) { - let mqtt_triggers = sqlx::query_as!( - MqttTrigger, - r#" - SELECT - mqtt_resource_path, - subscribe_topics as "subscribe_topics!: Vec>", - v3_config as "v3_config!: Option>", - v5_config as "v5_config!: Option>", - client_version as "client_version: _", - client_id, - workspace_id, - path, - script_path, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _" - FROM - mqtt_trigger - WHERE - enabled IS TRUE - AND (last_server_ping IS NULL OR - last_server_ping < now() - interval '15 seconds' - ) - "# - ) - .fetch_all(db) - .await; - - match mqtt_triggers { - Ok(mut triggers) => { - triggers.shuffle(&mut rand::rng()); - for trigger in triggers { - trigger - .try_to_listen_to_mqtt_messages(db.clone(), killpill_rx.resubscribe()) - .await; - } - } - Err(err) => { - tracing::error!("Error fetching mqtt triggers: {:?}", err); - } - }; - - let mqtt_triggers_capture = sqlx::query_as!( - CaptureConfigForMqttTrigger, - r#" - SELECT - path, - is_flow, - workspace_id, - owner, - email, - trigger_config as "trigger_config!: _" - FROM - capture_config - WHERE - trigger_kind = 'mqtt' AND - last_client_ping > NOW() - INTERVAL '10 seconds' AND - trigger_config IS NOT NULL AND - (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') - "# - ) - .fetch_all(db) - .await; - - match mqtt_triggers_capture { - Ok(mut captures) => { - captures.shuffle(&mut rand::rng()); - for capture in captures { - capture - .try_to_listen_to_mqtt_messages(db.clone(), killpill_rx.resubscribe()) - .await; - } - } - Err(err) => { - tracing::error!("Error fetching captures mqtt triggers: {:?}", err); - } - }; -} - -pub fn start_mqtt_consumer(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) { - tokio::spawn(async move { - listen_to_unlistened_mqtt_events(&db, &killpill_rx).await; - loop { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return; - } - _ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => { - listen_to_unlistened_mqtt_events(&db, &killpill_rx).await - } - } - } - }); -} diff --git a/backend/windmill-api/src/nats_triggers_oss.rs b/backend/windmill-api/src/nats_triggers_oss.rs deleted file mode 100644 index 53e8fd88ec..0000000000 --- a/backend/windmill-api/src/nats_triggers_oss.rs +++ /dev/null @@ -1,54 +0,0 @@ -#[cfg(feature = "private")] -#[allow(unused)] -pub use crate::nats_triggers_ee::*; - -#[cfg(not(feature = "private"))] -use crate::db::DB; -#[cfg(not(feature = "private"))] -use serde::{Deserialize, Serialize}; - -#[cfg(not(feature = "private"))] -#[derive(Serialize, Deserialize)] -pub struct NatsResourceAuth {} - -#[cfg(not(feature = "private"))] -pub fn start_nats_consumers(_db: DB, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>) -> () { - // implementation is not open source -} - -#[derive(Serialize, Deserialize)] -#[cfg(not(feature = "private"))] -pub enum NatsTriggerConfigConnection {} - -#[derive(Serialize, Clone)] -#[cfg(not(feature = "private"))] -pub struct NatsTrigger { - pub workspace_id: String, - pub path: String, - pub nats_resource_path: String, - pub subjects: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub stream_name: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub consumer_name: Option, - pub use_jetstream: bool, - pub script_path: String, - pub is_flow: bool, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - #[serde(skip_serializing_if = "Option::is_none")] - pub server_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_server_ping: Option>, - pub extra_perms: serde_json::Value, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option, -} diff --git a/backend/windmill-api/src/openapi.rs b/backend/windmill-api/src/openapi.rs index 6cc950523a..04d17a6127 100644 --- a/backend/windmill-api/src/openapi.rs +++ b/backend/windmill-api/src/openapi.rs @@ -9,6 +9,7 @@ use axum::{ }; use http::{header, HeaderValue, Method, StatusCode}; use indexmap::IndexMap; +use itertools::Itertools; use serde::{Deserialize, Serialize}; use serde_json::{to_value, Map, Value}; use sqlx::PgConnection; @@ -20,18 +21,13 @@ use windmill_common::{ DB, }; -use crate::db::ApiAuthed; - -#[cfg(feature = "http_trigger")] -use { - crate::{ - resources::try_get_resource_from_db_as, - triggers::http::{ - http_trigger_args::HttpMethod, http_trigger_auth::ApiKeyAuthentication, - AuthenticationMethod, - }, +use crate::{ + db::ApiAuthed, + resources::try_get_resource_from_db_as, + triggers::http::{ + http_trigger_args::HttpMethod, http_trigger_auth::ApiKeyAuthentication, + AuthenticationMethod, RequestType, }, - itertools::Itertools, }; lazy_static::lazy_static! { @@ -49,6 +45,7 @@ const BASIC_HTTP_AUTH_SCHEME: &'static str = "BasicHttp"; const DEFAULT_REQUEST_KEY: &'static str = "defaultRequest"; const DEFAULT_ASYNC_RESPONSE_KEY: &'static str = "AsyncResponse"; const DEFAULT_SYNC_RESPONSE_KEY: &'static str = "SyncResponse"; +const DEFAULT_SYNC_SSE_RESPONSE_KEY: &'static str = "SyncSseResponse"; const DEFAULT_PAYLOAD_PARAM_KEY: &'static str = "PayloadParam"; pub fn openapi_service() -> Router { @@ -166,7 +163,7 @@ pub enum Kind { pub struct FuturePath { route_path: String, kind: Kind, - is_async: Option, + request_type: Option, summary: Option, description: Option, security_scheme: Option, @@ -176,12 +173,12 @@ impl FuturePath { pub fn new( route_path: String, kind: Kind, - is_async: Option, + request_type: Option, summary: Option, description: Option, security_scheme: Option, ) -> FuturePath { - FuturePath { route_path, kind, is_async, summary, description, security_scheme } + FuturePath { route_path, kind, request_type, summary, description, security_scheme } } } @@ -241,6 +238,7 @@ fn from_route_path_to_openapi_path( vec![ format!("/run/{}", &normalized_path), format!("/run_wait_result/{}", &normalized_path), + format!("/run_and_stream/{}", &normalized_path), ] }; @@ -282,19 +280,23 @@ fn generate_paths( }) }; - let generate_response = |is_async: bool| { - let responses = if is_async { - serde_json::json!({ + let generate_response = |request_type: RequestType| { + let responses = match request_type { + RequestType::Async => serde_json::json!({ "200": { "$ref": format!("#/components/responses/{DEFAULT_ASYNC_RESPONSE_KEY}") } - }) - } else { - serde_json::json!(serde_json::json!({ + }), + RequestType::Sync => serde_json::json!({ "200": { "$ref": format!("#/components/responses/{DEFAULT_SYNC_RESPONSE_KEY}") } - })) + }), + RequestType::SyncSse => serde_json::json!({ + "200": { + "$ref": format!("#/components/responses/{DEFAULT_SYNC_SSE_RESPONSE_KEY}") + } + }), }; responses @@ -347,12 +349,19 @@ fn generate_paths( path_object }); - let is_async; + let request_type; let (methods, is_webhook) = match &path.kind { Kind::Webhook(_) => { - is_async = route_path.starts_with("/run/"); - let methods = if is_async { + request_type = if route_path.starts_with("/run/") { + RequestType::Async + } else if route_path.starts_with("/run_and_stream/") { + RequestType::SyncSse + } else { + RequestType::Sync + }; + + let methods = if request_type == RequestType::Async { vec![Method::POST] } else { vec![Method::GET, Method::POST] @@ -369,7 +378,7 @@ fn generate_paths( ) .into()); } - is_async = path.is_async.unwrap_or(true); + request_type = path.request_type.unwrap_or(RequestType::Sync); (vec![method.to_owned()], false) } }; @@ -401,7 +410,7 @@ fn generate_paths( ); } - method_map.insert("responses", generate_response(is_async)); + method_map.insert("responses", generate_response(request_type)); path_object.insert(method.to_string().to_lowercase(), to_value(&method_map)?); } @@ -428,19 +437,6 @@ pub fn transform_to_minified_postgres_regex(glob: &str) -> String { regex } -#[derive(Debug, Default)] -pub struct ServerToSet { - pub http_route: bool, - pub webhook_flow: bool, - pub webhook_script: bool, -} - -impl ServerToSet { - pub fn new(http_route: bool, webhook_flow: bool, webhook_script: bool) -> ServerToSet { - ServerToSet { http_route, webhook_flow, webhook_script } - } -} - fn header_to_pascal_case(header: &str) -> String { header .split(|c: char| c == '-' || c == '_' || c == ' ') @@ -582,7 +578,17 @@ fn generate_components(future_paths: &[FuturePath]) -> Map { "application/octet-stream": {} } }, - + DEFAULT_SYNC_SSE_RESPONSE_KEY: { + "description": "Returns an SSE stream.", + "content": { + "text/event-stream": { + "schema": { + "type": "string", + "description": "Stream of SSE" + }, + } + } + } })); components @@ -652,7 +658,6 @@ struct GenerateOpenAPI { openapi_spec_format: Format, } -#[cfg(feature = "http_trigger")] async fn http_routes_to_future_paths( db: &DB, user_db: UserDB, @@ -683,7 +688,7 @@ async fn http_routes_to_future_paths( struct MinifiedHttpTrigger { route_path: String, http_method: HttpMethod, - is_async: bool, + request_type: RequestType, workspaced_route: bool, summary: Option, description: Option, @@ -697,7 +702,7 @@ async fn http_routes_to_future_paths( SELECT route_path, http_method AS "http_method: _", - is_async, + request_type AS "request_type: _", workspaced_route, summary, description, @@ -765,7 +770,7 @@ async fn http_routes_to_future_paths( let future_path = FuturePath::new( route_path, Kind::HttpRoute(HttpRouteConfig::new(method)), - Some(http_route.is_async), + Some(http_route.request_type), http_route.summary, http_route.description, auth_method, @@ -777,18 +782,6 @@ async fn http_routes_to_future_paths( Ok(openapi_future_paths) } -#[cfg(not(feature = "http_trigger"))] -async fn http_routes_to_future_paths( - _db: &DB, - _user_db: UserDB, - _authed: &ApiAuthed, - _pg_pool: &mut PgConnection, - _http_route_filters: Option<&[HttpRouteFilter]>, - _w_id: &str, -) -> Result> { - Ok(Vec::new()) -} - async fn webhook_to_future_paths( pg_pool: &mut PgConnection, webhook_filters: Option<&[WebhookFilter]>, diff --git a/backend/windmill-api/src/postgres_triggers/mod.rs b/backend/windmill-api/src/postgres_triggers/mod.rs deleted file mode 100644 index 17d6d44678..0000000000 --- a/backend/windmill-api/src/postgres_triggers/mod.rs +++ /dev/null @@ -1,244 +0,0 @@ -use crate::{ - db::{ApiAuthed, DB}, - resources::try_get_resource_from_db_as, -}; -use native_tls::{Certificate, TlsConnector}; -use pg_escape::quote_identifier; -use rust_postgres::{config::SslMode, Client, Config, NoTls}; -use rust_postgres_native_tls::MakeTlsConnector; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sqlx::FromRow; -use std::collections::HashMap; - -use windmill_common::{ - db::UserDB, - error::{to_anyhow, Error, Result}, - utils::empty_as_none, -}; -mod bool; -mod converter; -mod hex; -mod relation; -mod replication_message; -mod trigger; - -pub use trigger::start_database; - -const ERROR_REPLICATION_SLOT_NOT_EXISTS: &str = r#"The replication slot associated with this trigger no longer exists. Recreate a new replication slot or select an existing one in the advanced tab, or delete and recreate a new trigger"#; - -const ERROR_PUBLICATION_NAME_NOT_EXISTS: &str = r#"The publication associated with this trigger no longer exists. Recreate a new publication or select an existing one in the advanced tab, or delete and recreate a new trigger"#; - -fn build_tls_connector( - ssl_mode: SslMode, - root_certificate_pem: Option<&String>, -) -> Result> { - let get_tls_builder_for_verify = |root_certificate: Option<&String>| { - let mut builder = TlsConnector::builder(); - if let Some(root_certificate) = root_certificate { - let root_certificate_pem = - Certificate::from_pem(root_certificate.as_bytes()).map_err(to_anyhow)?; - builder.add_root_certificate(root_certificate_pem); - } - Ok::<_, Error>(builder) - }; - let connector = match ssl_mode { - SslMode::Disable => return Ok(None), - SslMode::Require | SslMode::Prefer => { - let mut builder = TlsConnector::builder(); - builder.danger_accept_invalid_certs(true); - builder.danger_accept_invalid_hostnames(true); - builder - } - - SslMode::VerifyCa => { - let mut builder = get_tls_builder_for_verify(root_certificate_pem)?; - builder.danger_accept_invalid_hostnames(true); - builder - } - - SslMode::VerifyFull => { - let builder = get_tls_builder_for_verify(root_certificate_pem)?; - builder - } - _ => unreachable!(), - }; - - Ok(Some(MakeTlsConnector::new( - connector.build().map_err(to_anyhow)?, - ))) -} - -#[derive(FromRow, Serialize, Deserialize, Debug)] -pub struct Postgres { - pub user: String, - pub password: String, - pub host: String, - pub port: Option, - pub dbname: String, - #[serde(default)] - pub sslmode: String, - #[serde(default, deserialize_with = "empty_as_none")] - pub root_certificate_pem: Option, -} - -pub async fn get_raw_postgres_connection( - database: &Postgres, - logical_mode: bool, -) -> Result { - let ssl_mode = match database.sslmode.as_ref() { - "disable" => SslMode::Disable, - "" | "prefer" | "allow" => SslMode::Prefer, - "require" => SslMode::Require, - "verify-ca" => SslMode::VerifyCa, - "verify-full" => SslMode::VerifyFull, - ssl_mode => { - return Err(Error::BadRequest( - format!("Invalid ssl mode for postgres: {}, please put a valid ssl_mode among the following available ssl mode: ['disable', 'allow', 'prefer', 'verify-ca', 'verify-full']", ssl_mode), - )) - } - }; - - let mut config = Config::new(); - config - .dbname(&database.dbname) - .host(&database.host) - .user(&database.user) - .ssl_mode(ssl_mode); - - if logical_mode { - config.replication_mode(rust_postgres::config::ReplicationMode::Logical); - } - - if let Some(port) = database.port { - config.port(port); - }; - - if !database.password.is_empty() { - config.password(&database.password); - } - - let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?; - - let client = if let Some(connector) = connector { - let (client, connection) = config.connect(connector).await.map_err(to_anyhow)?; - tokio::spawn(async move { - if let Err(e) = connection.await { - tracing::debug!("{:#?}", e); - }; - tracing::info!("Successfully Connected into database"); - }); - client - } else { - let (client, connection) = config.connect(NoTls).await.map_err(to_anyhow)?; - tokio::spawn(async move { - if let Err(e) = connection.await { - tracing::debug!("{:#?}", e); - }; - tracing::info!("Successfully Connected into database"); - }); - client - }; - - Ok(client) -} - -pub async fn get_pg_connection( - authed: ApiAuthed, - user_db: Option, - db: &DB, - postgres_resource_path: &str, - w_id: &str, - logical_mode: bool, -) -> Result { - let database = - try_get_resource_from_db_as::(&authed, user_db, db, postgres_resource_path, w_id) - .await?; - - Ok(get_raw_postgres_connection(&database, logical_mode).await?) -} - -pub async fn get_default_pg_connection( - authed: ApiAuthed, - user_db: Option, - db: &DB, - postgres_resource_path: &str, - w_id: &str, -) -> Result { - get_pg_connection(authed, user_db, db, postgres_resource_path, w_id, false).await -} - -#[derive(FromRow, Deserialize, Serialize, Debug)] -pub struct PostgresTrigger { - pub path: String, - pub script_path: String, - pub is_flow: bool, - pub workspace_id: String, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - #[serde(skip_serializing_if = "Option::is_none")] - pub extra_perms: Option, - pub postgres_resource_path: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub server_id: Option, - pub replication_slot_name: String, - pub publication_name: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_server_ping: Option>, - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, -} - -pub async fn drop_logical_replication_slot(pg_connection: &Client, slot_name: &str) -> Result<()> { - let row = pg_connection - .query_opt( - r#" - SELECT - active_pid - FROM - pg_replication_slots - WHERE - slot_name = $1 - "#, - &[&slot_name], - ) - .await - .map_err(to_anyhow)?; - - let active_pid = row.map(|r| r.get::<_, Option>(0)).flatten(); - - if let Some(pid) = active_pid { - pg_connection - .execute("SELECT pg_terminate_backend($1)", &[&pid]) - .await - .map_err(to_anyhow)?; - } - - pg_connection - .execute("SELECT pg_drop_replication_slot($1)", &[&slot_name]) - .await - .map_err(to_anyhow)?; - - Ok(()) -} - -pub async fn drop_publication(pg_connection: &Client, publication_name: &str) -> Result<()> { - let mut query = String::from("DROP PUBLICATION IF EXISTS "); - let quoted_publication_name = quote_identifier(publication_name); - query.push_str("ed_publication_name); - - pg_connection - .execute(&query, &[]) - .await - .map_err(to_anyhow)?; - - Ok(()) -} diff --git a/backend/windmill-api/src/postgres_triggers/trigger.rs b/backend/windmill-api/src/postgres_triggers/trigger.rs deleted file mode 100644 index 6bd8877fec..0000000000 --- a/backend/windmill-api/src/postgres_triggers/trigger.rs +++ /dev/null @@ -1,1038 +0,0 @@ -use std::{collections::HashMap, pin::Pin}; - -use crate::{ - capture::{insert_capture_payload, PostgresTriggerConfig}, - db::{ApiAuthed, DB}, - postgres_triggers::{ - relation::RelationConverter, - replication_message::{ - LogicalReplicationMessage::{Begin, Commit, Delete, Insert, Relation, Type, Update}, - ReplicationMessage, - }, - }, - resources::try_get_resource_from_db_as, - trigger_helpers::{trigger_runnable, TriggerJobArgs}, - users::fetch_api_authed, -}; - -use bytes::{BufMut, Bytes, BytesMut}; -use chrono::TimeZone; -use futures::{pin_mut, SinkExt, StreamExt}; -use pg_escape::{quote_identifier, quote_literal}; -use rand::seq::SliceRandom; -use rust_postgres::{Client, CopyBothDuplex, SimpleQueryMessage}; -use serde::Deserialize; -use serde_json::value::RawValue; -use sqlx::types::Json as SqlxJson; - -use windmill_common::{ - db::UserDB, - error::{self, to_anyhow}, - triggers::TriggerKind, - utils::report_critical_error, - worker::to_raw_value, - INSTANCE_NAME, -}; - -use super::{ - drop_logical_replication_slot, drop_publication, get_default_pg_connection, - get_raw_postgres_connection, replication_message::PrimaryKeepAliveBody, Error, Postgres, - PostgresTrigger, ERROR_PUBLICATION_NAME_NOT_EXISTS, ERROR_REPLICATION_SLOT_NOT_EXISTS, -}; - -pub struct LogicalReplicationSettings { - pub streaming: bool, -} - -impl LogicalReplicationSettings { - pub fn new(streaming: bool) -> Self { - Self { streaming } - } -} - -#[allow(unused)] -trait RowExist { - fn row_exist(&self) -> bool; -} - -impl RowExist for Vec { - fn row_exist(&self) -> bool { - self.iter() - .find_map(|element| { - if let SimpleQueryMessage::CommandComplete(value) = element { - Some(*value) - } else { - None - } - }) - .is_some_and(|value| value > 0) - } -} - -pub struct PostgresSimpleClient(Client); - -impl PostgresSimpleClient { - async fn new(database: &Postgres) -> Result { - let client = get_raw_postgres_connection(database, true).await?; - - Ok(PostgresSimpleClient(client)) - } - - async fn execute_query( - &self, - query: &str, - ) -> Result, rust_postgres::Error> { - self.0.simple_query(query).await - } - - async fn get_logical_replication_stream( - &self, - publication_name: &str, - logical_replication_slot_name: &str, - ) -> Result<(CopyBothDuplex, LogicalReplicationSettings), Error> { - let options = format!( - r#"("proto_version" '2', "publication_names" {})"#, - quote_literal(publication_name), - ); - - let query = format!( - r#"START_REPLICATION SLOT {} LOGICAL 0/0 {}"#, - quote_identifier(logical_replication_slot_name), - options - ); - - Ok(( - self.0 - .copy_both_simple::(query.as_str()) - .await - .map_err(to_anyhow)?, - LogicalReplicationSettings::new(false), - )) - } - - async fn send_status_update( - primary_keep_alive: PrimaryKeepAliveBody, - copy_both_stream: &mut Pin<&mut CopyBothDuplex>, - ) { - let mut buf = BytesMut::new(); - let ts = chrono::Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); - let ts = chrono::Utc::now() - .signed_duration_since(ts) - .num_microseconds() - .unwrap_or(0); - - buf.put_u8(b'r'); - buf.put_u64(primary_keep_alive.wal_end); - buf.put_u64(primary_keep_alive.wal_end); - buf.put_u64(primary_keep_alive.wal_end); - buf.put_i64(ts); - buf.put_u8(0); - copy_both_stream.send(buf.freeze()).await.unwrap(); - tracing::debug!("Send update status message"); - } -} - -async fn loop_ping(db: &DB, pg: &PostgresConfig, error: Option<&str>) { - loop { - if pg.update_ping(db, error).await.is_none() { - return; - } - - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; - } -} - -enum PostgresConfig { - Trigger(PostgresTrigger), - Capture(CaptureConfigForPostgresTrigger), -} - -impl PostgresTrigger { - async fn try_to_listen_to_database_transactions( - self, - db: DB, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> () { - let postgres_trigger = sqlx::query_scalar!( - r#" - UPDATE postgres_trigger - SET - server_id = $1, - last_server_ping = now(), - error = 'Connecting...' - WHERE - enabled IS TRUE - AND workspace_id = $2 - AND path = $3 - AND (last_server_ping IS NULL - OR last_server_ping < now() - INTERVAL '15 seconds' - ) - RETURNING true - "#, - *INSTANCE_NAME, - self.workspace_id, - self.path, - ) - .fetch_optional(&db) - .await; - match postgres_trigger { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tracing::info!("Spawning new task to listen_to_database_transaction"); - tokio::spawn(async move { - listen_to_transactions( - PostgresConfig::Trigger(self), - db.clone(), - killpill_rx, - ) - .await; - }); - } else { - tracing::info!("Postgres trigger {} already being listened to", self.path); - } - } - Err(err) => { - tracing::error!( - "Error acquiring lock for postgres trigger {}: {:?}", - self.path, - err - ); - } - }; - } - - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - let updated = sqlx::query_scalar!( - r#" - UPDATE - postgres_trigger - SET - last_server_ping = now(), - error = $1 - WHERE - workspace_id = $2 - AND path = $3 - AND server_id = $4 - AND enabled IS TRUE - RETURNING 1 - "#, - error, - &self.workspace_id, - &self.path, - *INSTANCE_NAME - ) - .fetch_optional(db) - .await; - - match updated { - Ok(updated) => { - if updated.flatten().is_none() { - // allow faster restart of database trigger - sqlx::query!( - r#" - UPDATE - postgres_trigger - SET - last_server_ping = NULL - WHERE - workspace_id = $1 - AND path = $2 - AND server_id IS NULL"#, - &self.workspace_id, - &self.path, - ) - .execute(db) - .await - .ok(); - tracing::info!( - "Postgres trigger {} changed, disabled, or deleted, stopping...", - self.path - ); - return None; - } - } - Err(err) => { - tracing::warn!( - "Error updating ping of postgres trigger {}: {:?}", - self.path, - err - ); - } - }; - - Some(()) - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - match sqlx::query!( - r#" - UPDATE - postgres_trigger - SET - enabled = FALSE, - error = $1, - server_id = NULL, - last_server_ping = NULL - WHERE - workspace_id = $2 AND - path = $3 - "#, - error, - self.workspace_id, - self.path, - ) - .execute(db) - .await - { - Ok(_) => { - report_critical_error( - format!( - "Disabling postgres trigger {} because of error: {}", - self.path, error - ), - db.clone(), - Some(&self.workspace_id), - None, - ) - .await; - } - Err(disable_err) => { - report_critical_error( - format!("Could not disable postgres trigger {} with err {}, disabling because of error {}", self.path, disable_err, error), - db.clone(), - Some(&self.workspace_id), - None, - ).await; - } - } - } - - async fn fetch_authed(&self, db: &DB) -> error::Result { - fetch_api_authed( - self.edited_by.clone(), - self.email.clone(), - &self.workspace_id, - db, - Some(format!("pg-{}", self.path)), - ) - .await - } - - async fn handle(&self, db: &DB, payload: HashMap>) -> () { - if let Err(err) = run_job(payload, db, self).await { - report_critical_error( - format!( - "Failed to trigger job from postgres {}: {:?}", - self.path, err - ), - db.clone(), - Some(&self.workspace_id), - None, - ) - .await; - }; - } -} - -impl TriggerJobArgs>> for PostgresTrigger { - fn v1_payload_fn(payload: HashMap>) -> HashMap> { - payload - } - - fn v2_payload_fn(payload: HashMap>) -> HashMap> { - payload - } - - fn trigger_kind() -> TriggerKind { - TriggerKind::Postgres - } -} - -async fn run_job( - payload: HashMap>, - db: &DB, - trigger: &PostgresTrigger, -) -> anyhow::Result<()> { - let args = PostgresTrigger::build_job_args( - &trigger.script_path, - trigger.is_flow, - &trigger.workspace_id, - db, - payload, - HashMap::new(), - ) - .await?; - - let authed = fetch_api_authed( - trigger.edited_by.clone(), - trigger.email.clone(), - &trigger.workspace_id, - db, - Some(format!("postgres-{}", trigger.path)), - ) - .await?; - - trigger_runnable( - db, - None, - authed, - &trigger.workspace_id, - &trigger.script_path, - trigger.is_flow, - args, - trigger.retry.as_ref(), - trigger.error_handler_path.as_deref(), - trigger.error_handler_args.as_ref(), - format!("postgres_trigger/{}", trigger.path), - ) - .await?; - - Ok(()) -} - -struct PgInfo<'a> { - postgres_resource_path: &'a str, - publication_name: &'a str, - replication_slot_name: &'a str, - workspace_id: &'a str, -} - -impl PostgresConfig { - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - match self { - PostgresConfig::Trigger(trigger) => trigger.update_ping(db, error).await, - PostgresConfig::Capture(capture) => capture.update_ping(db, error).await, - } - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - match self { - PostgresConfig::Trigger(trigger) => trigger.disable_with_error(&db, error).await, - PostgresConfig::Capture(capture) => capture.disable_with_error(db, error).await, - } - } - - fn retrieve_info(&self) -> PgInfo<'_> { - let postgres_resource_path; - let publication_name; - let replication_slot_name; - let workspace_id; - - match self { - PostgresConfig::Trigger(trigger) => { - postgres_resource_path = &trigger.postgres_resource_path; - publication_name = &trigger.publication_name; - replication_slot_name = &trigger.replication_slot_name; - workspace_id = &trigger.workspace_id; - } - PostgresConfig::Capture(capture) => { - postgres_resource_path = &capture.trigger_config.postgres_resource_path; - workspace_id = &capture.workspace_id; - publication_name = capture.trigger_config.publication_name.as_ref().unwrap(); - replication_slot_name = capture - .trigger_config - .replication_slot_name - .as_ref() - .unwrap(); - } - }; - - PgInfo { postgres_resource_path, replication_slot_name, workspace_id, publication_name } - } - - async fn start_logical_replication_streaming( - &self, - db: &DB, - ) -> std::result::Result<(CopyBothDuplex, LogicalReplicationSettings), Error> { - let PgInfo { - publication_name, - replication_slot_name, - workspace_id, - postgres_resource_path, - } = self.retrieve_info(); - - let authed = match self { - PostgresConfig::Trigger(trigger) => trigger.fetch_authed(db).await?, - PostgresConfig::Capture(capture) => capture.fetch_authed(db).await?, - }; - - let database = try_get_resource_from_db_as::( - &authed, - Some(UserDB::new(db.clone())), - &db, - postgres_resource_path, - workspace_id, - ) - .await?; - - let client = PostgresSimpleClient::new(&database).await?; - - let publication = client - .execute_query(&format!( - "SELECT pubname FROM pg_publication WHERE pubname = {}", - quote_literal(&publication_name) - )) - .await - .map_err(to_anyhow)?; - - if !publication.row_exist() { - return Err(Error::BadConfig( - ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(), - )); - } - - let replication_slot = client - .execute_query(&format!( - "SELECT slot_name FROM pg_replication_slots WHERE slot_name = {}", - quote_literal(&replication_slot_name) - )) - .await - .map_err(to_anyhow)?; - - if !replication_slot.row_exist() { - return Err(Error::BadConfig( - ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string(), - )); - } - - let (logical_replication_stream, logical_replication_settings) = client - .get_logical_replication_stream(&publication_name, &replication_slot_name) - .await - .map_err(to_anyhow)?; - - Ok((logical_replication_stream, logical_replication_settings)) - } - - fn get_path(&self) -> &str { - match self { - PostgresConfig::Trigger(trigger) => &trigger.path, - PostgresConfig::Capture(capture) => &capture.path, - } - } - - async fn handle(&self, db: &DB, payload: HashMap>) -> () { - match self { - PostgresConfig::Trigger(trigger) => trigger.handle(&db, payload).await, - PostgresConfig::Capture(capture) => capture.handle(&db, payload).await, - } - } - - async fn cleanup(&self, db: &DB) -> Result<(), Error> { - match self { - PostgresConfig::Trigger(_) => Ok(()), - PostgresConfig::Capture(capture) => { - let publication_name = capture.trigger_config.publication_name.as_ref().unwrap(); - let replication_slot_name = capture - .trigger_config - .replication_slot_name - .as_ref() - .unwrap(); - let postgres_resource_path = &capture.trigger_config.postgres_resource_path; - let workspace_id = &capture.workspace_id; - let authed = capture.fetch_authed(&db).await?; - - let user_db = UserDB::new(db.clone()); - - let mut pg_connection = get_default_pg_connection( - authed.clone(), - Some(user_db.clone()), - &db, - postgres_resource_path, - workspace_id, - ) - .await?; - - if capture.trigger_config.basic_mode.unwrap_or(false) { - drop_logical_replication_slot(&mut pg_connection, replication_slot_name) - .await?; - - drop_publication(&mut pg_connection, publication_name).await?; - } - - Ok(()) - } - } - } -} - -async fn listen_to_transactions( - pg: PostgresConfig, - db: DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - let _ = pg.cleanup(&db).await; - return; - } - _ = loop_ping(&db, &pg, Some("Connecting...")) => { - let _ = pg.cleanup(&db).await; - return; - } - result = pg.start_logical_replication_streaming(&db) => { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - let _ = pg.cleanup(&db).await; - return; - } - _ = loop_ping(&db, &pg, None) => { - let _ = pg.cleanup(&db).await; - return; - } - _ = { - async { - match result { - Ok((logical_replication_stream, logical_replication_settings)) => { - pin_mut!(logical_replication_stream); - let mut relations = RelationConverter::new(); - tracing::info!("Starting to listen for postgres trigger {}", pg.get_path()); - loop { - let message = logical_replication_stream.next().await; - - let message = match message { - Some(message) => message, - None => { - tracing::error!("Stream for postgres trigger {} closed", pg.get_path()); - if let None = pg.update_ping(&db, Some("Stream closed")).await { - return; - } - return; - } - }; - - - let message = match message { - Ok(message) => message, - Err(err) => { - let err = format!("Postgres trigger named {} had an error while receiving a message : {}", pg.get_path(), err.to_string()); - pg.disable_with_error(&db, err).await; - return; - } - }; - - let logical_message = match ReplicationMessage::parse(message) { - Ok(logical_message) => logical_message, - Err(err) => { - let err = format!("Postgres trigger named: {} had an error while parsing message: {}", pg.get_path(), err.to_string()); - pg.disable_with_error(&db, err).await; - return; - } - }; - - - match logical_message { - ReplicationMessage::PrimaryKeepAlive(primary_keep_alive) => { - if primary_keep_alive.reply { - PostgresSimpleClient::send_status_update(primary_keep_alive, &mut logical_replication_stream).await; - } - } - ReplicationMessage::XLogData(x_log_data) => { - let logical_replication_message = match x_log_data.parse(&logical_replication_settings) { - Ok(logical_replication_message) => logical_replication_message, - Err(err) => { - tracing::error!("Postgres trigger named: {} had an error while trying to parse incomming stream message: {}", pg.get_path(), err.to_string()); - continue; - } - }; - - let json = match logical_replication_message { - Relation(relation_body) => { - relations.add_relation(relation_body); - None - } - Begin | Type | Commit => { - None - } - Insert(insert) => { - Some((insert.o_id, Ok(None), relations.row_to_json((insert.o_id, insert.tuple)), "insert")) - } - Update(update) => { - let old_row = update.old_tuple.map(|old_tuple| relations.row_to_json((update.o_id, old_tuple))).transpose(); - let row = relations.row_to_json((update.o_id, update.new_tuple)); - Some((update.o_id, old_row, row, "update")) - } - Delete(delete) => { - let row = delete.old_tuple.unwrap_or_else(|| delete.key_tuple.unwrap()); - Some((delete.o_id, Ok(None), relations.row_to_json((delete.o_id, row)), "delete")) - } - }; - match json { - Some((o_id, Ok(old_row), Ok(row), transaction_type)) => { - let relation = match relations.get_relation(o_id) { - Ok(relation) => relation, - Err(err) => { - tracing::error!("Postgres trigger named: {}, error: {}", pg.get_path(), err.to_string()); - continue; - } - }; - let database_info = HashMap::from([ - ("schema_name".to_string(), to_raw_value(&relation.namespace)), - ("table_name".to_string(), to_raw_value(&relation.name)), - ("transaction_type".to_string(), to_raw_value(&transaction_type)), - ("old_row".to_string(), to_raw_value(&old_row)), - ("row".to_string(), to_raw_value(&row)), - ]); - - - let _ = pg.handle(&db, database_info).await; - } - Some((o_id, old_row, row, transaction_type)) => { - let relation = match relations.get_relation(o_id) { - Ok(relation) => relation, - Err(err) => { - tracing::error!("Postgres trigger named: {}, error: {}", pg.get_path(), err.to_string()); - continue; - } - }; - - if let Err(err) = old_row { - tracing::error!( - transaction_type = ?transaction_type, - schema = %relation.namespace, - table = %relation.name, - error = %err, - "Failed to decode OLD row for {} transaction on {}.{}", - transaction_type, - relation.namespace, - relation.name, - ); - } - - if let Err(err) = row { - tracing::error!( - transaction_type = ?transaction_type, - schema = %relation.namespace, - table = %relation.name, - error = %err, - "Failed to decode NEW row for {} transaction on {}.{}", - transaction_type, - relation.namespace, - relation.name, - ); - } - - } - _ => {} - } - - } - } - } - } - Err(err) => { - tracing::error!("Postgres trigger error while trying to start logical replication streaming: {}", &err); - pg.disable_with_error(&db, err.to_string()).await - } - } - } - } => { - let _ = pg.cleanup(&db).await; - return; - } - } - } - } -} - -#[derive(Deserialize)] -struct CaptureConfigForPostgresTrigger { - trigger_config: SqlxJson, - path: String, - is_flow: bool, - workspace_id: String, - owner: String, - email: String, -} - -impl CaptureConfigForPostgresTrigger { - async fn try_to_listen_to_database_transactions( - self, - db: DB, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> () { - match sqlx::query_scalar!( - r#" - UPDATE - capture_config - SET - server_id = $1, - last_server_ping = now(), - error = 'Connecting...' - WHERE - last_client_ping > NOW() - INTERVAL '10 seconds' AND - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = 'postgres' AND - (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') - RETURNING true - "#, - *INSTANCE_NAME, - self.workspace_id, - self.path, - self.is_flow, - ) - .fetch_optional(&db) - .await - { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tokio::spawn(listen_to_transactions( - PostgresConfig::Capture(self), - db, - killpill_rx, - )); - } else { - tracing::info!("Postgres {} already being listened to", self.path); - } - } - Err(err) => { - tracing::error!( - "Error acquiring lock for capture postgres {}: {:?}", - self.path, - err - ); - } - }; - } - - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - match sqlx::query_scalar!( - r#" - UPDATE - capture_config - SET - last_server_ping = now(), - error = $1 - WHERE - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = 'postgres' AND - server_id = $5 AND - last_client_ping > NOW() - INTERVAL '10 seconds' - RETURNING 1 - "#, - error, - self.workspace_id, - self.path, - self.is_flow, - *INSTANCE_NAME - ) - .fetch_optional(db) - .await - { - Ok(updated) => { - if updated.flatten().is_none() { - // allow faster restart of postgres capture - sqlx::query!( - r#"UPDATE - capture_config - SET - last_server_ping = NULL - WHERE - workspace_id = $1 AND - path = $2 AND - is_flow = $3 AND - trigger_kind = 'postgres' AND - server_id IS NULL - "#, - self.workspace_id, - self.path, - self.is_flow, - ) - .execute(db) - .await - .ok(); - tracing::info!( - "Postgres capture {} changed, disabled, or deleted, stopping...", - self.path - ); - return None; - } - } - Err(err) => { - tracing::warn!( - "Error updating ping of capture postgres {}: {:?}", - self.path, - err - ); - } - }; - - Some(()) - } - - async fn fetch_authed(&self, db: &DB) -> error::Result { - fetch_api_authed( - self.owner.clone(), - self.email.clone(), - &self.workspace_id, - db, - Some(format!("postgres-{}", self.get_trigger_path())), - ) - .await - } - - fn get_trigger_path(&self) -> String { - format!( - "{}-{}", - if self.is_flow { "flow" } else { "script" }, - self.path - ) - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - if let Err(err) = sqlx::query!( - r#" - UPDATE - capture_config - SET - error = $1, - server_id = NULL, - last_server_ping = NULL - WHERE - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = 'postgres' - "#, - error, - self.workspace_id, - self.path, - self.is_flow, - ) - .execute(db) - .await - { - tracing::error!("Could not disable postgres capture {} ({}) with err {}, disabling because of error {}", self.path, self.workspace_id, err, error); - } - } - - async fn handle(&self, db: &DB, payload: HashMap>) -> () { - let main_args = PostgresTrigger::build_job_args_v2(false, payload.clone(), HashMap::new()); - let preprocessor_args = PostgresTrigger::build_job_args_v2(true, payload, HashMap::new()); - if let Err(err) = insert_capture_payload( - db, - &self.workspace_id, - &self.path, - self.is_flow, - &TriggerKind::Postgres, - main_args, - preprocessor_args, - &self.owner, - ) - .await - { - tracing::error!("Error inserting capture payload: {:?}", err); - } - } -} - -async fn listen_to_unlistened_database_events( - db: &DB, - killpill_rx: &tokio::sync::broadcast::Receiver<()>, -) { - let postgres_triggers = sqlx::query_as!( - PostgresTrigger, - r#" - SELECT - workspace_id, - path, - script_path, - replication_slot_name, - publication_name, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - postgres_resource_path, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _" - FROM - postgres_trigger - WHERE - enabled IS TRUE - AND (last_server_ping IS NULL OR - last_server_ping < now() - interval '15 seconds' - ) - "# - ) - .fetch_all(db) - .await; - - match postgres_triggers { - Ok(mut triggers) => { - triggers.shuffle(&mut rand::rng()); - for trigger in triggers { - trigger - .try_to_listen_to_database_transactions(db.clone(), killpill_rx.resubscribe()) - .await; - } - } - Err(err) => { - tracing::error!("Error fetching postgres triggers: {:?}", err); - } - }; - - let postgres_triggers_capture = sqlx::query_as!( - CaptureConfigForPostgresTrigger, - r#" - SELECT - path, - is_flow, - workspace_id, - owner, - email, - trigger_config as "trigger_config!: _" - FROM - capture_config - WHERE - trigger_kind = 'postgres' AND - last_client_ping > NOW() - INTERVAL '10 seconds' AND - trigger_config IS NOT NULL AND - (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') - "# - ) - .fetch_all(db) - .await; - - match postgres_triggers_capture { - Ok(mut captures) => { - captures.shuffle(&mut rand::rng()); - for capture in captures { - capture - .try_to_listen_to_database_transactions(db.clone(), killpill_rx.resubscribe()) - .await; - } - } - Err(err) => { - tracing::error!("Error fetching captures postgres triggers: {:?}", err); - } - }; -} - -pub fn start_database(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) { - tokio::spawn(async move { - listen_to_unlistened_database_events(&db, &killpill_rx).await; - loop { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return; - } - _ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => { - listen_to_unlistened_database_events(&db, &killpill_rx).await - } - } - } - }); -} diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 087d31cdcf..ec1eafe08a 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -11,8 +11,9 @@ use std::collections::HashMap; use crate::{ db::{ApiAuthed, DB}, users::{maybe_refresh_folders, require_owner_of_path, Tokened}, - utils::{check_scopes, BulkDeleteRequest}, + utils::{check_scopes, require_super_admin, BulkDeleteRequest}, var_resource_cache::{cache_resource, get_cached_resource}, + variables::get_value_internal, webhook_util::{WebhookMessage, WebhookShared}, }; use axum::{ @@ -28,15 +29,19 @@ use serde::{Deserialize, Serialize}; use serde_json::{value::RawValue, Value}; use sql_builder::{bind::Bind, quote, SqlBuilder}; use sqlx::{FromRow, Postgres, Transaction}; +use std::process::Stdio; +use tokio::process::Command; use uuid::Uuid; use windmill_audit::audit_oss::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; use windmill_common::{ - db::{UserDB, UserDbWithOptAuthed}, - error::{Error, JsonResult, Result}, + db::{UserDB, UserDbWithAuthed, UserDbWithOptAuthed}, + error::{self, Error, JsonResult, Result}, + get_database_url, parse_postgres_url, utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, variables, - worker::CLOUD_HOSTED, + worker::{CLOUD_HOSTED, TMP_DIR}, + workspaces::get_ducklake_instance_pg_catalog_password, }; pub fn workspaced_service() -> Router { @@ -56,6 +61,7 @@ pub fn workspaced_service() -> Router { .route("/delete/*path", delete(delete_resource)) .route("/delete_bulk", delete(delete_resources_bulk)) .route("/create", post(create_resource)) + .route("/git_commit_hash/*path", get(get_git_commit_hash)) .route("/type/list", get(list_resource_types)) .route("/type/listnames", get(list_resource_types_names)) .route("/type/get/:name", get(get_resource_type)) @@ -67,6 +73,7 @@ pub fn workspaced_service() -> Router { get(file_resource_ext_to_resource_type), ) .route("/type/create", post(create_resource_type)) + .route("/mcp_tools/*path", get(get_mcp_tools)) } pub fn public_service() -> Router { @@ -462,6 +469,20 @@ pub async fn get_resource_value_interpolated_internal( token: &str, allow_cache: bool, ) -> Result> { + // This is a special syntax to help debugging ducklake catalogs stored in the instance + if let Some(dbname) = path.strip_prefix("INSTANCE_DUCKLAKE_CATALOG/") { + require_super_admin(db, &authed.email).await?; + let pg_creds = parse_postgres_url(&get_database_url().await?)?; + return Ok(Some(serde_json::json!({ + "dbname": dbname, + "host": pg_creds.host, + "port": pg_creds.port, + "user": "ducklake_user", + "sslmode": pg_creds.ssl_mode, + "password": get_ducklake_instance_pg_catalog_password(&db).await?, + }))); + } + if allow_cache { if let Some(cached_value) = get_cached_resource(&workspace, &path) { return Ok(Some(cached_value)); @@ -515,7 +536,8 @@ pub async fn transform_json_value<'c>( match v { Value::String(y) if y.starts_with("$var:") => { let path = y.strip_prefix("$var:").unwrap(); - let userdb_authed = UserDbWithOptAuthed { authed: authed, user_db: user_db.clone(), db: db.clone() }; + let userdb_authed = + UserDbWithOptAuthed { authed: authed, user_db: user_db.clone(), db: db.clone() }; let v = crate::variables::get_value_internal( &userdb_authed, @@ -616,6 +638,7 @@ pub async fn transform_json_value<'c>( job.root_job.map(|x| x.to_string()), Some(job.scheduled_for.clone()), None, + None, ) .await; @@ -719,7 +742,7 @@ async fn create_resource( "INSERT INTO resource (workspace_id, path, value, description, resource_type, created_by, edited_at) VALUES ($1, $2, $3, $4, $5, $6, now()) ON CONFLICT (workspace_id, path) - DO UPDATE SET value = $3, description = $4, resource_type = $5, edited_at = now()", + DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now()", w_id, resource.path, raw_json as sqlx::types::Json<&RawValue>, @@ -775,13 +798,14 @@ async fn delete_resource( check_scopes(&authed, || format!("resources:write:{}", path))?; let mut tx = user_db.begin(&authed).await?; - sqlx::query!( - "DELETE FROM resource WHERE path = $1 AND workspace_id = $2", + let deleted_path = sqlx::query_scalar!( + "DELETE FROM resource WHERE path = $1 AND workspace_id = $2 RETURNING path", path, w_id ) - .execute(&mut *tx) + .fetch_optional(&mut *tx) .await?; + not_found_if_none(deleted_path, "Resource", &path)?; sqlx::query!( "DELETE FROM variable WHERE path = $1 AND workspace_id = $2", path, @@ -1225,13 +1249,16 @@ async fn delete_resource_type( let mut tx = user_db.begin(&authed).await?; - sqlx::query!( - "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2", + let deleted_name = sqlx::query_scalar!( + "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2 RETURNING name", name, w_id ) - .execute(&mut *tx) + .fetch_optional(&mut *tx) .await?; + + not_found_if_none(deleted_name, "ResourceType", &name)?; + audit_log( &mut *tx, &authed, @@ -1369,3 +1396,280 @@ where Ok(resource) } + +/// Get list of tools from an MCP resource +async fn get_mcp_tools( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("resources:read:{}", path))?; + + let mut tx = user_db.begin(&authed).await?; + + // Fetch the MCP resource from database + let resource_value_o = sqlx::query_scalar!( + "SELECT value as \"value: sqlx::types::Json>\" FROM resource WHERE path = $1 AND workspace_id = $2", + &path, + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + + tx.commit().await?; + + if resource_value_o.is_none() { + explain_resource_perm_error(&path, &w_id, &db, &authed).await?; + } + + let resource_value = not_found_if_none(resource_value_o, "Resource", path)? + .ok_or_else(|| Error::BadRequest(format!("Empty resource value for {}", path)))?; + + // Parse MCP resource + let mcp_resource = + serde_json::from_str::(resource_value.0.get()) + .map_err(|e| Error::BadRequest(format!("Failed to parse MCP resource: {}", e)))?; + + // Create MCP client connection + let client = windmill_common::mcp_client::McpClient::from_resource(mcp_resource, &db, &w_id) + .await + .map_err(|e| Error::ExecutionErr(format!("Failed to connect to MCP server: {}", e)))?; + + // Get raw MCP tools and convert to JSON + let tools: Vec = client + .available_tools() + .iter() + .map(|tool| { + serde_json::to_value(tool) + .map_err(|e| Error::ExecutionErr(format!("Failed to serialize MCP tool: {}", e))) + }) + .collect::>>()?; + + // Gracefully shutdown the client + if let Err(e) = client.shutdown().await { + tracing::warn!("Failed to shutdown MCP client: {}", e); + } + + Ok(Json(tools)) +} + +#[derive(Deserialize, Serialize)] +struct GitRepositoryResource { + url: String, + #[serde(skip_serializing_if = "Option::is_none")] + branch: Option, +} + +#[derive(Serialize)] +struct GitCommitHashResponse { + commit_hash: String, +} + +#[derive(Deserialize)] +struct GitCommitHashQuery { + git_ssh_identity: Option, +} + +async fn get_git_commit_hash( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Tokened { token }: Tokened, + Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, +) -> JsonResult { + let path = path.to_path(); + + check_scopes(&authed, || format!("resources:read:{}", path))?; + + let git_repo_resource_value = get_resource_value_interpolated_internal( + &authed, + Some(user_db.clone()), + &db, + &w_id, + path, + None, + &token, + false, + ) + .await + .map_err(|e| Error::NotFound(format!("Access to resource {} denied: ({e})", path)))?; + + let git_resource: GitRepositoryResource = match git_repo_resource_value { + Some(value) => serde_json::from_value(value).map_err(|e| { + Error::BadRequest(format!("Invalid git repository resource format: {}", e)) + })?, + None => return Err(Error::NotFound(format!("Resource {} not found", path)).into()), + }; + + let identities: Vec = query + .git_ssh_identity + .map(|s| { + s.split(",") + .filter_map(|s| { + if !s.is_empty() { + Some(s.to_string()) + } else { + None + } + }) + .collect() + }) + .unwrap_or(vec![]); + + let (git_ssh_cmd, filenames) = + get_git_ssh_cmd(&authed, &user_db, &db, &w_id, identities).await?; + + let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd).await; + + delete_paths(&filenames).await; + + Ok(Json(GitCommitHashResponse { commit_hash: commit_hash? })) +} + +async fn write_ssh_file( + authed: &ApiAuthed, + user_db: &UserDB, + db: &DB, + w_id: &str, + var_path: &str, +) -> std::result::Result { + let id_file_name = format!(".ssh_id_priv_{}", Uuid::new_v4()); + let loc = std::path::Path::new(TMP_DIR) + .join("ssh_ids") + .join(id_file_name); + + let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; + let mut content = get_value_internal(&userdb_authed, db, w_id, var_path, authed, false) + .await + .map_err(|e| { + ( + error::Error::NotFound(format!( + "Variable {var_path} not found for git ssh identity: {e:#}" + )), + loc.clone(), + ) + })?; + content.push_str("\n"); + + if let Some(p) = &loc.parent() { + tokio::fs::create_dir_all(p) + .await + .map_err(|e| (e.into(), loc.clone()))?; + } + tokio::fs::write(&loc, content) + .await + .map_err(|e| (e.into(), loc.clone()))?; + + #[cfg(unix)] + { + let perm = std::os::unix::fs::PermissionsExt::from_mode(0o600); + tokio::fs::set_permissions(&loc, perm) + .await + .map_err(|e| (e.into(), loc.clone()))?; + } + + return Ok(loc); +} + +async fn delete_paths(paths: &Vec) { + for path in paths { + let _ = tokio::fs::remove_file(&path).await; + } +} + +async fn get_git_ssh_cmd( + authed: &ApiAuthed, + user_db: &UserDB, + db: &DB, + w_id: &str, + git_ssh_identity: Vec, +) -> error::Result<(Option, Vec)> { + if git_ssh_identity.len() > 5 { + return Err(error::Error::BadRequest( + "Too many ssh identities, try using at most 1".to_string(), + )); + } + if git_ssh_identity.len() == 0 { + return Ok((None, vec![])); + } + + let mut ssh_id_files = vec![]; + let mut file_paths = vec![]; + for var_path in git_ssh_identity.iter() { + match write_ssh_file(authed, user_db, db, w_id, &var_path).await { + Ok(loc) => { + ssh_id_files.push(format!( + " -i '{}'", + loc.to_string_lossy().replace('\'', r"'\''") + )); + file_paths.push(loc); + } + Err((e, loc)) => { + file_paths.push(loc); + delete_paths(&file_paths).await; + return Err(e); + } + } + } + + let git_ssh_cmd = format!("ssh -o StrictHostKeyChecking=no{}", ssh_id_files.join("")); + Ok((Some(git_ssh_cmd), file_paths)) +} + +async fn get_repo_latest_commit_hash( + git_resource: &GitRepositoryResource, + git_ssh_command: Option, +) -> Result { + let mut git_cmd = Command::new("git"); + + let ref_spec = git_resource + .branch + .as_deref() + .filter(|s| !s.is_empty()) + .unwrap_or("HEAD"); + + git_cmd.args(["ls-remote", &git_resource.url, ref_spec]); + if let Some(git_ssh_command) = git_ssh_command { + git_cmd.env("GIT_SSH_COMMAND", git_ssh_command); + } + git_cmd.stderr(Stdio::piped()); + + let output = git_cmd + .output() + .await + .map_err(|e| Error::internal_err(format!("Failed to execute git command: {}", e)))?; + + if !output.status.success() { + let stderr = String::from_utf8(output.stderr) + .unwrap_or_else(|_| "Failed to decode stderr".to_string()); + return Err(Error::BadRequest(format!( + "Error getting git repo commit hash: {}", + stderr + ))); + } + + let stdout = String::from_utf8(output.stdout) + .map_err(|e| Error::internal_err(format!("Failed to decode git output: {}", e)))?; + + let lines: Vec<&str> = stdout.lines().collect(); + + if lines.is_empty() { + return Err(Error::BadRequest(format!( + "No commits found for reference '{}' in repository '{}'", + ref_spec, git_resource.url + ))); + } + + let commit_hash = lines + .first() + .and_then(|line| line.split_whitespace().next()) + .map(|s| s.to_string()) + .ok_or_else(|| { + Error::BadRequest("Unexpected output format for git ls-remote".to_string()) + })?; + + Ok(commit_hash) +} diff --git a/backend/windmill-api/src/schedule.rs b/backend/windmill-api/src/schedule.rs index b5bb69f8ee..134ea46246 100644 --- a/backend/windmill-api/src/schedule.rs +++ b/backend/windmill-api/src/schedule.rs @@ -78,6 +78,7 @@ pub struct NewSchedule { pub tag: Option, pub paused_until: Option>, pub cron_version: Option, + pub dynamic_skip: Option, } #[derive(Serialize, Deserialize)] @@ -128,6 +129,35 @@ fn to_json_raw_opt( value.map(|v| sqlx::types::Json(to_raw_value(&v))) } +/// Validate that a dynamic skip handler (script or flow) exists +async fn validate_dynamic_skip<'c>( + tx: &mut Transaction<'c, Postgres>, + w_id: &str, + handler_path: &str, +) -> Result<()> { + // Check for script only (flows are not supported in the UI) + let exists = sqlx::query_scalar!( + "SELECT EXISTS( + SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 AND archived = false AND deleted = false + )", + w_id, + handler_path + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(false); + + if exists { + Ok(()) + } else { + Err(Error::BadRequest(format!( + "Dynamic skip handler '{}' not found. The handler must be an existing, non-archived script at schedule creation time.", + handler_path + ))) + } +} + async fn create_schedule( authed: ApiAuthed, Extension(db): Extension, @@ -169,6 +199,11 @@ async fn create_schedule( check_path_conflict(&mut tx, &w_id, &ns.path).await?; check_flow_conflict(&mut tx, &w_id, &ns.path, ns.is_flow, &ns.script_path).await?; + // Validate dynamic_skip if provided + if let Some(handler_path) = &ns.dynamic_skip { + validate_dynamic_skip(&mut tx, &w_id, handler_path).await?; + } + let schedule = sqlx::query_as!( Schedule, r#" @@ -179,7 +214,7 @@ async fn create_schedule( on_recovery, on_recovery_times, on_recovery_extra_args, on_success, on_success_extra_args, ws_error_handler_muted, retry, summary, no_flow_overlap, - tag, paused_until, cron_version, description + tag, paused_until, cron_version, description, dynamic_skip ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, @@ -187,7 +222,7 @@ async fn create_schedule( $15, $16, $17, $18, $19, $20, $21, $22, $23, - $24, $25, $26, $27 + $24, $25, $26, $27, $28 ) RETURNING workspace_id, @@ -219,7 +254,8 @@ async fn create_schedule( description, tag, paused_until, - cron_version + cron_version, + dynamic_skip "#, w_id, ns.path, @@ -251,7 +287,8 @@ async fn create_schedule( ns.tag, ns.paused_until, ns.cron_version.clone().unwrap_or_else(|| "v2".to_string()), - ns.description + ns.description, + ns.dynamic_skip ) .fetch_one(&mut *tx) .await @@ -277,7 +314,7 @@ async fn create_schedule( .await?; if ns.enabled.unwrap_or(true) { - tx = push_scheduled_job(&db, tx, &schedule, Some(&authed.clone().into())).await? + tx = push_scheduled_job(&db, tx, &schedule, Some(&authed.clone().into()), None).await? } tx.commit().await?; @@ -311,6 +348,11 @@ async fn edit_schedule( // Check schedule for error ScheduleType::from_str(&es.schedule, es.cron_version.as_deref(), true)?; + // Validate dynamic_skip if provided + if let Some(handler_path) = &es.dynamic_skip { + validate_dynamic_skip(&mut tx, &w_id, handler_path).await?; + } + clear_schedule(&mut tx, path, &w_id).await?; let schedule = sqlx::query_as!( Schedule, @@ -337,7 +379,8 @@ async fn edit_schedule( path = $19, workspace_id = $20, cron_version = COALESCE($21, cron_version), - description = $22 + description = $22, + dynamic_skip = $23 WHERE path = $19 AND workspace_id = $20 RETURNING workspace_id, @@ -369,7 +412,8 @@ async fn edit_schedule( description, tag, paused_until, - cron_version + cron_version, + dynamic_skip "#, es.schedule, es.timezone, @@ -396,7 +440,8 @@ async fn edit_schedule( path, w_id, es.cron_version, - es.description + es.description, + es.dynamic_skip ) .fetch_one(&mut *tx) .await @@ -419,7 +464,7 @@ async fn edit_schedule( .await?; if schedule.enabled { - tx = push_scheduled_job(&db, tx, &schedule, None).await?; + tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; } tx.commit().await?; @@ -538,7 +583,7 @@ async fn list_schedule_with_jobs( AND j.workspace_id = $1 AND parent_job IS NULL AND runnable_path = schedule.script_path AND status <> 'skipped' - ORDER BY created_at DESC + ORDER BY completed_at DESC LIMIT 20 ) AS jobs) t WHERE workspace_id = $1 @@ -657,7 +702,8 @@ pub async fn set_enabled( description, tag, paused_until, - cron_version + cron_version, + dynamic_skip "#, payload.enabled, authed.email, @@ -683,7 +729,7 @@ pub async fn set_enabled( .await?; if payload.enabled { - tx = push_scheduled_job(&db, tx, &schedule, None).await?; + tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; } tx.commit().await?; @@ -1006,6 +1052,7 @@ pub struct EditSchedule { pub tag: Option, pub paused_until: Option>, pub cron_version: Option, + pub dynamic_skip: Option, } pub async fn clear_schedule<'c>( diff --git a/backend/windmill-api/src/scopes.rs b/backend/windmill-api/src/scopes.rs index ef20131659..7984e4b627 100644 --- a/backend/windmill-api/src/scopes.rs +++ b/backend/windmill-api/src/scopes.rs @@ -484,21 +484,24 @@ pub fn check_route_access( ))) } -const SCRIPT_JOBS: [&'static str; 6] = [ +const SCRIPT_JOBS: [&'static str; 8] = [ "jobs/run/p", "jobs/run/h", "jobs/run_wait_result/p", "jobs/run_wait_result/h", "jobs/run/preview_bundle", "jobs/run/preview", + "jobs/run_and_stream/p", + "jobs/run_and_stream/h", ]; -const FLOW_JOBS: [&'static str; 5] = [ +const FLOW_JOBS: [&'static str; 6] = [ "jobs/run/f", "jobs/run_wait_result/f", "jobs/run/preview_flow", "jobs/restart/f", "jobs/flow/resume", + "jobs/run_and_stream/f", ]; lazy_static::lazy_static! { diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 355acde53b..d491d46792 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -26,6 +26,7 @@ use axum::{ Json, Router, }; use futures::future::try_join_all; +use http::header; use hyper::StatusCode; use itertools::Itertools; use quick_cache::sync::Cache; @@ -37,14 +38,15 @@ use sqlx::{FromRow, Postgres, Transaction}; use std::{collections::HashMap, sync::Arc}; use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; -use windmill_worker::process_relative_imports; +use windmill_worker::{process_relative_imports, scoped_dependency_map::ScopedDependencyMap}; use windmill_common::{ assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind, AssetWithAltAccessType}, error::to_anyhow, + s3_helpers::upload_artifact_to_store, scripts::hash_script, utils::WarnAfterExt, - worker::CLOUD_HOSTED, + worker::{CLOUD_HOSTED, MIN_VERSION_SUPPORTS_DEBOUNCING}, }; use windmill_common::{ @@ -118,6 +120,10 @@ pub struct ScriptWDraft { #[serde(skip_serializing_if = "Option::is_none")] #[sqlx(json(nullable))] pub assets: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub debounce_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub debounce_delay_s: Option, } pub fn global_service() -> Router { @@ -125,6 +131,7 @@ pub fn global_service() -> Router { .route("/hub/top", get(get_top_hub_scripts)) .route("/hub/get/*path", get(get_hub_script_by_path)) .route("/hub/get_full/*path", get(get_full_hub_script_by_path)) + .route("/hub/pick/*path", get(pick_hub_script_by_path)) } pub fn global_unauthed_service() -> Router { @@ -386,6 +393,7 @@ async fn create_snapshot_script( Path(w_id): Path, mut multipart: Multipart, ) -> Result<(StatusCode, String)> { + // TODO: Check for debouncing here as well. let mut script_hash = None; let mut tx = None; let mut uploaded = false; @@ -420,45 +428,13 @@ async fn create_snapshot_script( uploaded = true; - #[cfg(all(feature = "enterprise", feature = "parquet"))] - let object_store = windmill_common::s3_helpers::get_object_store().await; - - #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - let object_store: Option<()> = None; - - if &windmill_common::utils::MODE_AND_ADDONS.mode - == &windmill_common::utils::Mode::Standalone - && object_store.is_none() - { - std::fs::create_dir_all( - windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR.clone(), - )?; - windmill_common::worker::write_file_bytes( - &windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR, - &hash, - &data, - )?; - } else { - #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - { - return Err(Error::ExecutionErr("codebase is an EE feature".to_string())); - } - - #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = object_store { - let path = windmill_common::s3_helpers::bundle(&w_id, &hash); - - if let Err(e) = os - .put(&object_store::path::Path::from(path.clone()), data.into()) - .await - { - tracing::info!("Failed to put snapshot to s3 at {path}: {:?}", e); - return Err(Error::ExecutionErr(format!("Failed to put {path} to s3"))); - } - } else { - return Err(Error::BadConfig("Object store is required for snapshot script and is not configured for servers".to_string())); - } - } + let path = windmill_common::s3_helpers::bundle(&w_id, &hash); + upload_artifact_to_store( + &path, + data, + &windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR, + ) + .await?; } // println!("Length of `{}` is {} bytes", name, data.len()); } @@ -550,6 +526,8 @@ async fn create_script_internal<'c>( )> { check_scopes(&authed, || format!("scripts:write:{}", ns.path))?; + guard_script_from_debounce_data(&ns).await?; + let codebase = ns.codebase.as_ref(); #[cfg(not(feature = "enterprise"))] if ns.ws_error_handler_muted.is_some_and(|val| val) { @@ -799,13 +777,21 @@ async fn create_script_internal<'c>( } }; + // Row lock debounce key for path. We need this to make all updates of runnables sequential and predictable. + tokio::time::timeout( + core::time::Duration::from_secs(60), + windmill_common::jobs::lock_debounce_key(&w_id, &ns.path, &mut tx), + ) + .warn_after_seconds(10) + .await??; + sqlx::query!( "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \ content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34)", + delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36)", &w_id, &hash.0, ns.path, @@ -843,10 +829,13 @@ async fn create_script_internal<'c>( None }, validate_schema, - ns.assets.as_ref().and_then(|a| serde_json::to_value(a).ok()) + ns.assets.as_ref().and_then(|a| serde_json::to_value(a).ok()), + ns.debounce_key, + ns.debounce_delay_s, ) .execute(&mut *tx) .await?; + let p_path_opt = parent_hashes_and_perms.as_ref().map(|x| x.p_path.clone()); if let Some(ref p_path) = p_path_opt { sqlx::query!( @@ -895,11 +884,21 @@ async fn create_script_internal<'c>( schedulables.push(schedule); } + // Update dynamic_skip references when script is renamed + sqlx::query!( + "UPDATE schedule SET dynamic_skip = $1 WHERE dynamic_skip = $2 AND workspace_id = $3", + &ns.path, + &p_path, + &w_id + ) + .execute(&mut *tx) + .await?; + for schedule in schedulables { clear_schedule(&mut tx, &schedule.path, &w_id).await?; if schedule.enabled { - tx = push_scheduled_job(&db, tx, &schedule, None).await?; + tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; } } } else { @@ -1019,6 +1018,8 @@ async fn create_script_internal<'c>( None, Some(&authed.clone().into()), false, + None, + None, ) .await?; Ok((hash, new_tx, None)) @@ -1035,6 +1036,9 @@ async fn create_script_internal<'c>( let content = ns.content.clone(); let language = ns.language.clone(); tokio::spawn(async move { + // TODO: I don't think we want this. We might want to send dependency job. But skip any calculations if lock is already present. + // It will allow us to make code more consistent and predictable. + // wait for 10 seconds to make sure the script is deployed and that the CLI sync that pushed it (f one) is complete tokio::time::sleep(std::time::Duration::from_secs(10)).await; if let Err(e) = process_relative_imports( @@ -1109,6 +1113,40 @@ pub async fn get_full_hub_script_by_path( )) } +pub async fn pick_hub_script_by_path( + Path(path): Path, + Extension(db): Extension, +) -> impl IntoResponse { + let path_str = path.to_path(); + + // Extract version_id from path (format: {hub}/{version_id}/{summary}) + let version_id = path_str.split('/').nth(1).unwrap_or(""); + + let hub_base_url = HUB_BASE_URL.read().await.clone(); + + // Determine which hub to use based on version_id + // If version_id < PRIVATE_HUB_MIN_VERSION, use default hub + let target_hub_url = if version_id + .parse::() + .is_ok_and(|v| v < windmill_common::PRIVATE_HUB_MIN_VERSION) + { + windmill_common::DEFAULT_HUB_BASE_URL + } else { + &hub_base_url + }; + + // Call the hub's pick endpoint: /scripts/{version_id}/pick + let (status_code, headers, response) = query_elems_from_hub( + &HTTP_CLIENT, + &format!("{}/scripts/{}/pick", target_hub_url, version_id), + None, + &db, + ) + .await?; + + Ok::<_, Error>((status_code, headers, response)) +} + async fn get_script_by_path( authed: ApiAuthed, Extension(user_db): Extension, @@ -1178,7 +1216,7 @@ async fn get_script_by_path_w_draft( let mut tx = user_db.begin(&authed).await?; let script_o = sqlx::query_as::<_, ScriptWDraft>( - "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email, assets FROM script LEFT JOIN draft ON + "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email, assets, debounce_key, debounce_delay_s FROM script LEFT JOIN draft ON script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script' WHERE script.path = $1 AND script.workspace_id = $2 ORDER BY script.created_at DESC LIMIT 1", @@ -1265,7 +1303,8 @@ async fn update_script_history( let mut tx = user_db.begin(&authed).await?; sqlx::query!( - "INSERT INTO deployment_metadata (workspace_id, path, script_hash, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL DO UPDATE SET deployment_msg = $4", + "INSERT INTO deployment_metadata (workspace_id, path, script_hash, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL + DO UPDATE SET deployment_msg = EXCLUDED.deployment_msg", w_id, script_path, script_hash.0, @@ -1367,7 +1406,7 @@ async fn get_tokened_raw_script_by_path( Extension(cache): Extension>, Path((w_id, token, path)): Path<(String, String, StripPath)>, Query(query): Query, -) -> Result { +) -> Result { let authed = cache .get_authed(Some(w_id.clone()), &token) .await @@ -1393,17 +1432,28 @@ struct RawScriptByPathQuery { // used specifically for python to cache folders on import success to avoid extra db calls on package fetch cache_folders: Option, } + +struct StringWithLength(String); + +impl IntoResponse for StringWithLength { + fn into_response(self) -> axum::response::Response { + let len = self.0.len(); + ([(header::CONTENT_LENGTH, len.to_string())], self.0).into_response() + } +} + async fn raw_script_by_path( authed: ApiAuthed, Extension(user_db): Extension, Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, Query(query): Query, -) -> Result { +) -> Result { if *DEBUG_RAW_SCRIPT_ENDPOINTS { tracing::warn!("Raw script by path request: {}", path.to_path()); } - raw_script_by_path_internal(path, user_db, db, authed, w_id, false, query).await + let r = raw_script_by_path_internal(path, user_db, db, authed, w_id, false, query).await?; + Ok(StringWithLength(r)) } async fn raw_script_by_path_unpinned( @@ -1412,8 +1462,9 @@ async fn raw_script_by_path_unpinned( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, Query(query): Query, -) -> Result { - raw_script_by_path_internal(path, user_db, db, authed, w_id, true, query).await +) -> Result { + let r = raw_script_by_path_internal(path, user_db, db, authed, w_id, true, query).await?; + Ok(StringWithLength(r)) } lazy_static::lazy_static! { @@ -1494,7 +1545,10 @@ async fn raw_script_by_path_internal( return Ok("WINDMILL_IS_FOLDER".to_string()); } else { if *DEBUG_RAW_SCRIPT_ENDPOINTS { - tracing::warn!("Raw script by path request: {} (cached folders expired)", path); + tracing::warn!( + "Raw script by path request: {} (cached folders expired)", + path + ); } } } @@ -1512,7 +1566,11 @@ async fn raw_script_by_path_internal( .await?; tx.commit().await?; if *DEBUG_RAW_SCRIPT_ENDPOINTS { - tracing::warn!("Raw script by path request: {} (content: {:?})", path, content_o); + tracing::warn!( + "Raw script by path request: {} (content: {:?})", + path, + content_o + ); } if content_o.is_none() { @@ -1762,7 +1820,11 @@ async fn archive_script_by_path( Some([("workspace", w_id.as_str())].into()), ) .await?; - tx.commit().await?; + + ScopedDependencyMap::clear_map_for_item(path, &w_id, "script", tx, &None) + .await + .commit() + .await?; handle_deployment_metadata( &authed.email, @@ -1796,9 +1858,10 @@ async fn archive_script_by_hash( let mut tx = user_db.begin(&authed).await?; let script = sqlx::query_as::<_, Script>( - "UPDATE script SET archived = true WHERE hash = $1 RETURNING *", + "UPDATE script SET archived = true WHERE hash = $1 AND workspace_id = $2 RETURNING *", ) .bind(&hash.0) + .bind(&w_id) .fetch_one(&mut *tx) .await .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; @@ -1822,7 +1885,11 @@ async fn archive_script_by_hash( Some([("workspace", w_id.as_str())].into()), ) .await?; - tx.commit().await?; + + ScopedDependencyMap::clear_map_for_item(&script.path, &w_id, "script", tx, &None) + .await + .commit() + .await?; webhook.send_message( w_id.clone(), @@ -2116,3 +2183,18 @@ async fn delete_scripts_bulk( Ok(Json(deleted_paths)) } + +/// Validates that script debouncing configuration is supported by all workers +/// Returns an error if debouncing is configured but workers are behind required version +async fn guard_script_from_debounce_data(ns: &NewScript) -> Result<()> { + if !*MIN_VERSION_SUPPORTS_DEBOUNCING.read().await + && (ns.debounce_key.is_some() || ns.debounce_delay_s.is_some()) + { + tracing::warn!( + "Script debouncing configuration rejected: workers are behind minimum required version for debouncing feature" + ); + Err(Error::WorkersAreBehind { feature: "Debouncing".into(), min_version: "1.566.0".into() }) + } else { + Ok(()) + } +} diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 73936466c0..4851130286 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -6,7 +6,7 @@ * LICENSE-AGPL for a copy of the license. */ -use std::time::Duration; +use std::{collections::HashMap, time::Duration}; use crate::{ db::{ApiAuthed, DB}, @@ -23,6 +23,7 @@ use axum::{ #[cfg(feature = "enterprise")] use axum::extract::Query; +use serde_json::json; #[cfg(feature = "enterprise")] use crate::utils::require_devops_role; @@ -42,7 +43,6 @@ use windmill_common::{ }, parse_postgres_url, server::Smtp, - utils::build_arg_str, }; pub fn global_service() -> Router { @@ -69,10 +69,13 @@ pub fn global_service() -> Router { "/critical_alerts/:id/acknowledge", post(acknowledge_critical_alert), ) - .route("/databases_exist", post(databases_exist)) .route( - "/create_ducklake_database/:name", - post(create_ducklake_database), + "/get_ducklake_instance_catalog_db_status", + post(get_ducklake_instance_catalog_db_status), + ) + .route( + "/setup_ducklake_catalog_db/:name", + post(setup_ducklake_catalog_db), ) .route( "/critical_alerts/acknowledge_all", @@ -215,6 +218,10 @@ pub struct Value { } pub async fn delete_global_setting(db: &DB, key: &str) -> error::Result<()> { + if key == "ducklake_user_pg_pwd" || key == "ducklake_settings" { + tracing::error!("Tried to unset global setting {}, ignored", key); + return Ok(()); + } sqlx::query!("DELETE FROM global_settings WHERE name = $1", key,) .execute(db) .await?; @@ -331,7 +338,7 @@ pub async fn set_global_setting_internal( } v => { sqlx::query!( - "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()", + "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", key, v ) @@ -356,6 +363,7 @@ pub async fn get_global_setting( && key != HUB_BASE_URL_SETTING && key != HUB_ACCESSIBLE_URL_SETTING && key != EMAIL_DOMAIN_SETTING + && key != APP_WORKSPACED_ROUTE_SETTING { require_super_admin(&db, &authed.email).await?; } @@ -569,91 +577,164 @@ pub async fn acknowledge_all_critical_alerts() -> error::Error { error::Error::NotFound("Critical Alerts require EE".to_string()) } -async fn databases_exist( - _authed: ApiAuthed, - Extension(db): Extension, - Json(database_names): Json>, -) -> JsonResult> { - let result = sqlx::query_scalar!( - r#"SELECT elem FROM (SELECT unnest($1::TEXT[]) AS elem) AS e - WHERE elem NOT IN (SELECT datname FROM pg_catalog.pg_database);"#, - database_names.as_slice() - ) - .fetch_all(&db) - .await? - .into_iter() - .filter_map(|x| x) - .collect(); - - Ok(Json(result)) +#[derive(Deserialize, Debug, Serialize)] +struct DucklakeInstanceCatalogDbStatus { + logs: DucklakeInstanceCatalogDbStatusLogs, // (Step, Message)[] + success: bool, + error: Option, } -async fn create_ducklake_database( +#[derive(Deserialize, Debug, Serialize, Default)] +#[serde(default)] +struct DucklakeInstanceCatalogDbStatusLogs { + super_admin: String, + #[serde(skip_serializing_if = "String::is_empty")] + database_credentials: String, + #[serde(skip_serializing_if = "String::is_empty")] + valid_dbname: String, + #[serde(skip_serializing_if = "String::is_empty")] + created_database: String, + #[serde(skip_serializing_if = "String::is_empty")] + db_connect: String, + #[serde(skip_serializing_if = "String::is_empty")] + grant_permissions: String, +} + +async fn get_ducklake_instance_catalog_db_status( + _authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + let result = sqlx::query_scalar!( + r#"SELECT value->'instance_catalog_db_status' FROM global_settings WHERE name = 'ducklake_settings'"#, + ) + .fetch_one(&db) + .await? + .ok_or_else(|| error::Error::ExecutionErr("Couldn't find ducklake_settings".to_string()))?; + let result = serde_json::from_value(result).map_err(|e| { + error::Error::ExecutionErr(format!( + "couldn't parse instance_catalog_db_status : {}", + e.to_string() + )) + })?; + return Ok(Json(result)); +} + +async fn setup_ducklake_catalog_db( authed: ApiAuthed, Extension(db): Extension, Path(dbname): Path, +) -> JsonResult { + let mut logs = DucklakeInstanceCatalogDbStatusLogs::default(); + let result = setup_ducklake_catalog_db_inner(authed, &db, &dbname, &mut logs).await; + let success = result.is_ok(); + let error = result.err().map(|e| e.to_string()); + let status = DucklakeInstanceCatalogDbStatus { logs, success, error }; + let status_json = serde_json::to_value(&status).map_err(to_anyhow)?; + // Save that the database was setup successfully + sqlx::query!( + r#"UPDATE global_settings SET value = jsonb_set(value, '{instance_catalog_db_status}', (COALESCE(value->'instance_catalog_db_status', '{}'::jsonb) || to_jsonb($1::json))) WHERE name = 'ducklake_settings'"#, + json!({ dbname: status_json }) + ).execute(&db).await?; + + Ok(Json(status)) +} + +async fn setup_ducklake_catalog_db_inner( + authed: ApiAuthed, + db: &DB, + dbname: &str, + logs: &mut DucklakeInstanceCatalogDbStatusLogs, ) -> Result<()> { - require_super_admin(&db, &authed.email).await?; + require_super_admin(db, &authed.email).await?; + logs.super_admin = "OK".to_string(); + let pg_creds = &get_database_url().await?; + let pg_creds = parse_postgres_url(pg_creds)?; + logs.database_credentials = "OK".to_string(); // Validate name to ensure it only contains alphanumeric characters // Prevents SQL injection on the instance database - let valid_name = regex::Regex::new(r"^[a-zA-Z0-9_]+$") - .map_err(|_| error::Error::internal_err("Failed to compile regex".to_string()))?; - if !valid_name.is_match(&dbname) { + lazy_static::lazy_static! { + static ref VALID_NAME: regex::Regex = regex::Regex::new(r"^[a-zA-Z0-9_]+$").unwrap(); + } + if !VALID_NAME.is_match(dbname) { return Err(error::Error::BadRequest( - "Invalid database name".to_string(), + "Catalog name must be alphanumeric, underscores allowed".to_string(), )); } + if pg_creds.database.trim().eq_ignore_ascii_case(dbname.trim()) { + return Err(error::Error::BadRequest( + "Database name cannot be the same as the main database".to_string(), + )); + } + logs.valid_dbname = "OK".to_string(); - sqlx::query(&format!("CREATE DATABASE \"{dbname}\"")) - .execute(&db) - .await?; + let db_exists = sqlx::query_scalar!( + "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", + dbname + ) + .fetch_one(db) + .await? + .unwrap_or(false); - sqlx::query(&format!( - "GRANT CONNECT ON DATABASE \"{dbname}\" TO ducklake_user" - )) - .execute(&db) - .await?; + logs.created_database = "SKIP".to_string(); + if !db_exists { + sqlx::query(&format!("CREATE DATABASE \"{dbname}\"")) + .execute(db) + .await?; + logs.created_database = "OK".to_string(); + } - // We have to connect to the newly created database as admin to grant permissions - let pg_creds = parse_postgres_url(&get_database_url().await?)?; - let Some(wm_pg_pwd) = pg_creds.password else { - return Err(error::Error::BadRequest("Password not found".to_string())); + let ssl_mode = match pg_creds.ssl_mode.as_deref() { + Some("allow") => "prefer".to_string(), + Some("verify-ca") | Some("verify-full") => "require".to_string(), + Some(s) => s.to_string(), + None => "prefer".to_string(), }; - let conn_str: String = build_arg_str( - &[ - ("host", Some(&pg_creds.host)), - ("port", pg_creds.port.map(|p| p.to_string()).as_deref()), - ("password", Some(&wm_pg_pwd)), - ("user", pg_creds.username.as_deref()), - ("dbname", Some(&dbname)), - ], - " ", - "=", + // We have to connect to the newly created database as admin to grant permissions + let conn_str = format!( + "postgres://{user}:{password}@{host}:{port}/{dbname}?sslmode={sslmode}", + user = urlencoding::encode(&pg_creds.username.unwrap_or_else(|| "postgres".to_string())), + password = urlencoding::encode(&pg_creds.password.as_deref().unwrap_or("")), + host = urlencoding::encode(&pg_creds.host), + port = pg_creds.port.unwrap_or(5432), + dbname = dbname, + sslmode = ssl_mode ); + let (client, connection) = tokio::time::timeout( std::time::Duration::from_secs(20), tokio_postgres::connect(&conn_str, tokio_postgres::NoTls), ) .await - .map_err(to_anyhow)? - .map_err(to_anyhow)?; - - tokio::spawn(async move { - if let Err(e) = connection.await { - eprintln!("connection error: {}", e); - } - }); + .map_err(|e| error::Error::ExecutionErr(format!("timeout: {}", e.to_string())))? + .map_err(|e| error::Error::ExecutionErr(format!("error: {}", e.to_string())))?; + let join_handle = tokio::spawn(async move { connection.await }); + logs.db_connect = "OK".to_string(); client .batch_execute(&format!( - "GRANT USAGE ON SCHEMA public TO ducklake_user; + "GRANT CONNECT ON DATABASE \"{dbname}\" TO ducklake_user; + GRANT USAGE ON SCHEMA public TO ducklake_user; GRANT CREATE ON SCHEMA public TO ducklake_user; ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ducklake_user;" )) .await - .map_err(to_anyhow)?; + .map_err(|e| { + error::Error::ExecutionErr(format!( + "Failed to grant permissions to ducklake_user: {}", + e.to_string(), + )) + })?; + logs.grant_permissions = "OK".to_string(); + + drop(client); // /!\ Drop before joining to avoid deadlock + join_handle + .await + .map_err(|e| error::Error::ExecutionErr(format!("join error: {}", e.to_string())))? + .map_err(|e| { + error::Error::ExecutionErr(format!("tokio_postgres error: {}", e.to_string())) + })?; Ok(()) } diff --git a/backend/windmill-api/src/sqs_triggers_oss.rs b/backend/windmill-api/src/sqs_triggers_oss.rs deleted file mode 100644 index 49fffb43e9..0000000000 --- a/backend/windmill-api/src/sqs_triggers_oss.rs +++ /dev/null @@ -1,44 +0,0 @@ -#[cfg(feature = "private")] -#[allow(unused)] -pub use crate::sqs_triggers_ee::*; - -#[cfg(not(feature = "private"))] -use crate::db::DB; -#[cfg(not(feature = "private"))] -use serde::{Deserialize, Serialize}; -#[cfg(not(feature = "private"))] -use windmill_common::auth::aws::AwsAuthResourceType; - -#[cfg(not(feature = "private"))] -pub fn start_sqs(_db: DB, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>) -> () { - // implementation is not open source -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[cfg(not(feature = "private"))] -pub struct SqsTrigger { - pub queue_url: String, - pub aws_auth_resource_type: AwsAuthResourceType, - pub aws_resource_path: String, - pub message_attributes: Option>, - pub path: String, - pub script_path: String, - pub is_flow: bool, - pub workspace_id: String, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - pub extra_perms: Option, - pub error: Option, - pub server_id: Option, - pub last_server_ping: Option>, - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option< - sqlx::types::Json>>, - >, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, -} diff --git a/backend/windmill-api/src/teams_cache_oss.rs b/backend/windmill-api/src/teams_cache_oss.rs new file mode 100644 index 0000000000..9501c16f00 --- /dev/null +++ b/backend/windmill-api/src/teams_cache_oss.rs @@ -0,0 +1,3 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::teams_cache_ee::*; diff --git a/backend/windmill-api/src/teams_oss.rs b/backend/windmill-api/src/teams_oss.rs index 95d4883690..037e556c99 100644 --- a/backend/windmill-api/src/teams_oss.rs +++ b/backend/windmill-api/src/teams_oss.rs @@ -23,6 +23,13 @@ pub async fn workspaces_list_available_teams_ids() -> Result )); } +#[cfg(not(feature = "private"))] +pub async fn workspaces_list_available_teams_channels() -> Result { + return Err(Error::BadRequest( + "Teams only available on enterprise".to_string(), + )); +} + #[cfg(not(feature = "private"))] pub async fn connect_teams() -> Result { return Err(Error::BadRequest( @@ -37,12 +44,6 @@ pub async fn run_teams_message_test_job() -> Result { )); } -#[cfg(not(feature = "private"))] -pub async fn workspaces_list_available_teams_channels() -> Result { - return Err(Error::BadRequest( - "Teams only available on enterprise".to_string(), - )); -} #[cfg(all(feature = "enterprise", not(feature = "private")))] pub fn teams_service() -> Router { diff --git a/backend/windmill-api/src/tracing_init.rs b/backend/windmill-api/src/tracing_init.rs index c5c841b2dc..8ae3eabf47 100644 --- a/backend/windmill-api/src/tracing_init.rs +++ b/backend/windmill-api/src/tracing_init.rs @@ -46,8 +46,8 @@ impl OnResponse for MyOnResponse { pub struct MyOnFailure {} impl OnFailure for MyOnFailure { - fn on_failure(&mut self, _b: B, _latency: std::time::Duration, _span: &tracing::Span) { - // tracing::error!(latency = latency.as_millis(), "response") + fn on_failure(&mut self, _b: B, latency: std::time::Duration, _span: &tracing::Span) { + tracing::error!(latency = latency.as_millis(), "response failure") } } diff --git a/backend/windmill-api/src/trigger_helpers.rs b/backend/windmill-api/src/trigger_helpers.rs deleted file mode 100644 index 14f9e42005..0000000000 --- a/backend/windmill-api/src/trigger_helpers.rs +++ /dev/null @@ -1,808 +0,0 @@ -use anyhow::Context; -use axum::response::IntoResponse; -use http::StatusCode; -use serde::Deserialize; -use serde_json::value::RawValue; -use std::collections::HashMap; -use uuid::Uuid; -use windmill_common::{ - db::{UserDB, UserDbWithAuthed}, - error::Result, - flows::{FlowModuleValue, Retry}, - get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, - jobs::{get_has_preprocessor_from_content_and_lang, script_path_to_payload, JobPayload}, - scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, - triggers::{ - HubOrWorkspaceId, RunnableFormat, RunnableFormatVersion, TriggerKind, - RUNNABLE_FORMAT_VERSION_CACHE, - }, - users::username_to_permissioned_as, - utils::StripPath, - worker::to_raw_value, - FlowVersionInfo, -}; -use windmill_queue::{push, PushArgs, PushArgsOwned, PushIsolationLevel}; - -#[cfg(feature = "enterprise")] -use crate::jobs::check_license_key_valid; -use crate::{ - db::{ApiAuthed, DB}, - jobs::{ - check_tag_available_for_workspace, delete_job_metadata_after_use, result_to_response, - run_flow_by_path_inner, run_script_by_path_inner, run_wait_result_internal, RunJobQuery, - }, - utils::check_scopes, - HTTP_CLIENT, -}; - -struct ScriptInfo { - has_preprocessor: Option, - language: ScriptLang, - content: String, - schema: Option>, -} - -#[derive(Debug, Deserialize)] -struct PropertyDefinition { - r#type: Option, -} - -#[derive(Debug, Deserialize)] -struct PartialSchema { - properties: Option>, -} - -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub enum RunnableId { - FlowPath(String), - ScriptId(ScriptId), - HubScript(String), -} - -impl RunnableId { - #[allow(dead_code)] - pub fn from_script_hash(hash: ScriptHash) -> Self { - Self::ScriptId(ScriptId::ScriptHash(hash)) - } - - pub fn from_script_path(path: &str) -> Self { - if path.starts_with("hub/") { - Self::HubScript(path.to_string()) - } else { - Self::ScriptId(ScriptId::ScriptPath(path.to_string())) - } - } - - pub fn from_flow_path(path: &str) -> Self { - Self::FlowPath(path.to_string()) - } -} - -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -#[allow(unused)] -pub enum ScriptId { - ScriptPath(String), - ScriptHash(ScriptHash), -} - -impl ScriptId { - async fn get_script_hash(self, workspace_id: &str, db: &DB) -> Result { - let hash = match self { - ScriptId::ScriptPath(path) => { - let info = get_latest_deployed_hash_for_path(None, db.clone(), workspace_id, &path) - .await?; - info.hash - } - ScriptId::ScriptHash(hash) => hash.0, - }; - - Ok(hash) - } -} - -async fn get_script_info( - db: &DB, - workspace_id: &str, - hash: i64, -) -> std::result::Result { - sqlx::query_as!(ScriptInfo, "SELECT has_preprocessor, language as \"language: _\", content, schema as \"schema: _\" FROM script WHERE workspace_id = $1 AND hash = $2", workspace_id, hash) - .fetch_one(db) - .await -} - -fn runnable_format_from_schema_without_preprocessor( - trigger_kind: &TriggerKind, - schema: Option>, -) -> RunnableFormat { - match trigger_kind { - TriggerKind::Mqtt - if schema.as_ref().is_some_and(|schema| { - schema.properties.as_ref().is_some_and(|properties| { - properties.iter().any(|(key, def)| { - key == "payload" && def.r#type.as_ref().is_some_and(|t| t == "array") - }) - }) - }) => - { - RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false } - } - TriggerKind::Kafka | TriggerKind::Nats - if schema.as_ref().is_some_and(|schema| { - schema - .properties - .as_ref() - .is_some_and(|properties| properties.keys().any(|key| key == "msg")) - }) => - { - RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false } - } - _ => RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: false }, - } -} - -fn runnable_format_from_preprocessor_args( - args: Option>, -) -> RunnableFormat { - if let Some(args) = args { - if args.iter().any(|arg| arg.name == "wm_trigger") - || (args.len() > 0 && args.iter().all(|arg| arg.name != "event")) - { - RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true } - } else { - RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true } - } - } else { - RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true } - } -} - -enum PreprocessorInfo { - Preprocessor { content: String, language: ScriptLang }, - NoPreprocessor { schema: Option> }, -} - -#[derive(Debug, Deserialize)] -struct FlowInfo { - preprocessor_module: Option>, - schema: Option>, -} - -fn get_preprocessor_args_from_content_and_language( - content: &str, - language: &ScriptLang, -) -> Result>> { - let args = match language { - ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno | ScriptLang::Nativets => { - let args = windmill_parser_ts::parse_deno_signature( - &content, - true, - false, - Some("preprocessor".to_string()), - )?; - Some(args.args) - } - ScriptLang::Python3 => { - let args = windmill_parser_py::parse_python_signature( - &content, - Some("preprocessor".to_string()), - false, - )?; - Some(args.args) - } - _ => None, - }; - Ok(args) -} - -pub async fn get_runnable_format( - runnable_id: RunnableId, - workspace_id: &str, - db: &DB, - trigger_kind: &TriggerKind, -) -> Result { - let (key, preprocessor_info) = match runnable_id { - RunnableId::HubScript(path) => { - let Some(version) = path.split("/").nth(1) else { - return Err(windmill_common::error::Error::internal_err( - "Invalid hub script path".to_string(), - )); - }; - - let version = match version.parse::() { - Ok(version) => version, - Err(_) => { - return Err(windmill_common::error::Error::internal_err( - "Invalid hub script version".to_string(), - )); - } - }; - - let key = (HubOrWorkspaceId::Hub, version, trigger_kind.clone()); - - let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); - - if let Some(runnable_format) = runnable_format { - tracing::debug!("Using cached runnable format for hub script {path}"); - return Ok(runnable_format); - } - - let hub_script = - get_full_hub_script_by_path(StripPath(path.to_string()), &HTTP_CLIENT, Some(db)) - .await?; - - let has_preprocessor = get_has_preprocessor_from_content_and_lang( - &hub_script.content, - &hub_script.language, - )?; - - let partial_schema = serde_json::from_str(hub_script.schema.get())?; - - ( - key, - if has_preprocessor { - PreprocessorInfo::Preprocessor { - content: hub_script.content, - language: hub_script.language, - } - } else { - PreprocessorInfo::NoPreprocessor { - schema: Some(sqlx::types::Json(partial_schema)), - } - }, - ) - } - RunnableId::FlowPath(path) => { - let FlowVersionInfo { version, .. } = - get_latest_flow_version_info_for_path(None, &db, workspace_id, &path, true).await?; - - let key = ( - HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()), - version, - trigger_kind.clone(), - ); - - let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); - - if let Some(runnable_format) = runnable_format { - tracing::debug!("Using cached runnable format for flow {path}"); - return Ok(runnable_format); - } - - let flow_info = sqlx::query_as!( - FlowInfo, - "SELECT - value->'preprocessor_module'->'value' as \"preprocessor_module: _\", - schema as \"schema: _\" - FROM flow_version - WHERE - path = $1 - AND workspace_id = $2 - ORDER BY created_at DESC - LIMIT 1", - path, - workspace_id, - ) - .fetch_one(db) - .await?; - - if let Some(preprocessor_module) = flow_info.preprocessor_module { - match preprocessor_module.0 { - FlowModuleValue::RawScript { content, language, .. } => { - (key, PreprocessorInfo::Preprocessor { content, language }) - } - FlowModuleValue::Script { path, hash, .. } => { - let hash = if let Some(hash) = hash { - hash.0 - } else { - let script_hash = get_latest_deployed_hash_for_path( - None, - db.clone(), - workspace_id, - &path, - ) - .await?; - script_hash.hash - }; - let script_info = get_script_info(db, workspace_id, hash).await?; - ( - key, - PreprocessorInfo::Preprocessor { - content: script_info.content, - language: script_info.language, - }, - ) - } - _ => { - return Err(windmill_common::error::Error::internal_err( - "Unsupported preprocessor module".to_string(), - )); - } - } - } else { - ( - key, - PreprocessorInfo::NoPreprocessor { schema: flow_info.schema }, - ) - } - } - RunnableId::ScriptId(script_id) => { - let hash = script_id.get_script_hash(workspace_id, db).await?; - let key = ( - HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()), - hash, - trigger_kind.clone(), - ); - let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); - - if let Some(runnable_format) = runnable_format { - tracing::debug!("Using cached runnable format for script {hash}"); - return Ok(runnable_format); - } - - let script_info = get_script_info(db, workspace_id, hash).await?; - - if script_info.has_preprocessor.unwrap_or(false) { - ( - key, - PreprocessorInfo::Preprocessor { - content: script_info.content, - language: script_info.language, - }, - ) - } else { - ( - key, - PreprocessorInfo::NoPreprocessor { schema: script_info.schema }, - ) - } - } - }; - - let runnable_format = match preprocessor_info { - PreprocessorInfo::Preprocessor { content, language } => { - let args = get_preprocessor_args_from_content_and_language(&content, &language)?; - runnable_format_from_preprocessor_args(args) - } - PreprocessorInfo::NoPreprocessor { schema } => { - runnable_format_from_schema_without_preprocessor(trigger_kind, schema) - } - }; - - RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format); - - Ok(runnable_format) -} - -#[allow(dead_code)] -pub trait TriggerJobArgs { - fn v1_payload_fn(payload: T) -> HashMap>; - fn v2_payload_fn(payload: T) -> HashMap> { - Self::v1_payload_fn(payload) - } - fn trigger_kind() -> TriggerKind; - - fn build_job_args_v2( - has_preprocessor: bool, - payload: T, - info: HashMap>, - ) -> PushArgsOwned { - let trigger_kind = Self::trigger_kind(); - let mut args = Self::v2_payload_fn(payload); - if has_preprocessor { - args.insert("kind".to_string(), to_raw_value(&trigger_kind.to_key())); - args.extend(info); - let args = HashMap::from([("event".to_string(), to_raw_value(&args))]); - PushArgsOwned { args, extra: None } - } else { - PushArgsOwned { args, extra: None } - } - } - - fn build_job_args_v1( - has_preprocessor: bool, - payload: T, - info: HashMap>, - ) -> PushArgsOwned { - let trigger_kind = Self::trigger_kind(); - let trigger_key = trigger_kind.to_key(); - let args = Self::v1_payload_fn(payload); - let extra = if has_preprocessor { - Some(HashMap::from([( - "wm_trigger".to_string(), - to_raw_value(&serde_json::json!({ - "kind": trigger_key, - trigger_key: info - })), - )])) - } else { - None - }; - - PushArgsOwned { args, extra } - } - - async fn build_job_args( - runnable_path: &str, - is_flow: bool, - w_id: &str, - db: &DB, - payload: T, - info: HashMap>, - ) -> Result { - let runnable_id = if is_flow { - RunnableId::from_flow_path(runnable_path) - } else { - RunnableId::from_script_path(runnable_path) - }; - Self::build_job_args_from_runnable_id(runnable_id, w_id, db, payload, info).await - } - - async fn build_job_args_from_runnable_id( - runnable_id: RunnableId, - w_id: &str, - db: &DB, - payload: T, - info: HashMap>, - ) -> Result { - let runnable_format = - get_runnable_format(runnable_id, w_id, db, &Self::trigger_kind()).await?; - - match runnable_format { - RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor } => { - Ok(Self::build_job_args_v1(has_preprocessor, payload, info)) - } - RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor } => { - Ok(Self::build_job_args_v2(has_preprocessor, payload, info)) - } - } - } - - fn build_capture_payloads( - payload: T, - info: HashMap>, - ) -> (PushArgsOwned, PushArgsOwned) { - let main_args = Self::build_job_args_v2(false, payload.clone(), info.clone()); - let preprocessor_args = Self::build_job_args_v2(true, payload, info); - (main_args, preprocessor_args) - } -} - -#[allow(dead_code)] -async fn trigger_runnable_inner( - db: &DB, - user_db: Option, - authed: ApiAuthed, - workspace_id: &str, - runnable_path: &str, - is_flow: bool, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>>, - trigger_path: String, -) -> Result<(Uuid, Option, Option)> { - let user_db = user_db.unwrap_or_else(|| UserDB::new(db.clone())); - let (uuid, delete_after_use, early_return) = if is_flow { - let run_query = RunJobQuery::default(); - let path = StripPath(runnable_path.to_string()); - let (uuid, early_return) = run_flow_by_path_inner( - authed, - db.clone(), - user_db, - workspace_id.to_string(), - path, - run_query, - args, - ) - .await?; - (uuid, None, early_return) - } else { - let (uuid, delete_after_use) = trigger_script_internal( - db, - user_db, - authed, - workspace_id, - runnable_path, - args, - retry, - error_handler_path, - error_handler_args, - trigger_path, - ) - .await?; - (uuid, delete_after_use, None) - }; - - Ok((uuid, delete_after_use, early_return)) -} - -#[allow(dead_code)] -pub async fn trigger_runnable( - db: &DB, - user_db: Option, - authed: ApiAuthed, - workspace_id: &str, - runnable_path: &str, - is_flow: bool, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>>, - trigger_path: String, -) -> Result { - let (uuid, _, _) = trigger_runnable_inner( - db, - user_db, - authed, - workspace_id, - runnable_path, - is_flow, - args, - retry, - error_handler_path, - error_handler_args, - trigger_path, - ) - .await?; - Ok((StatusCode::CREATED, uuid.to_string()).into_response()) -} - -#[allow(dead_code)] -pub async fn trigger_runnable_and_wait_for_result( - db: &DB, - user_db: Option, - authed: ApiAuthed, - workspace_id: &str, - runnable_path: &str, - is_flow: bool, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>>, - trigger_path: String, -) -> Result { - let username = authed.username.clone(); - let (uuid, delete_after_use, early_return) = trigger_runnable_inner( - db, - user_db, - authed, - workspace_id, - runnable_path, - is_flow, - args, - retry, - error_handler_path, - error_handler_args, - trigger_path, - ) - .await?; - let (result, success) = - run_wait_result_internal(db, uuid, workspace_id.to_string(), early_return, &username) - .await?; - - if delete_after_use.unwrap_or(false) { - delete_job_metadata_after_use(&db, uuid).await?; - } - - result_to_response(result, success) -} - -#[allow(dead_code)] -pub async fn trigger_runnable_and_wait_for_raw_result( - db: &DB, - user_db: Option, - authed: ApiAuthed, - workspace_id: &str, - runnable_path: &str, - is_flow: bool, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>>, - trigger_path: String, -) -> Result> { - let username = authed.username.clone(); - let (uuid, delete_after_use, early_return) = trigger_runnable_inner( - db, - user_db, - authed, - workspace_id, - runnable_path, - is_flow, - args, - retry, - error_handler_path, - error_handler_args, - trigger_path, - ) - .await?; - - let (result, success) = - run_wait_result_internal(db, uuid, workspace_id.to_string(), early_return, &username) - .await - .with_context(|| { - format!( - "Error fetching job result for {} {}", - if is_flow { "flow" } else { "script" }, - runnable_path - ) - })?; - - if delete_after_use.unwrap_or(false) { - delete_job_metadata_after_use(&db, uuid).await?; - } - - if !success { - Err(windmill_common::error::Error::internal_err(format!( - "{} {runnable_path} failed: {:?}", - if is_flow { "Flow" } else { "Script" }, - result - ))) - } else { - Ok(result) - } -} - -async fn trigger_script_internal( - db: &DB, - user_db: UserDB, - authed: ApiAuthed, - workspace_id: &str, - script_path: &str, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>>, - trigger_path: String, -) -> Result<(Uuid, Option)> { - if retry.is_none() && error_handler_path.is_none() { - let run_query = RunJobQuery::default(); - let path = StripPath(script_path.to_string()); - run_script_by_path_inner( - authed, - db.clone(), - user_db, - workspace_id.to_string(), - path, - run_query, - args, - ) - .await - } else { - trigger_script_with_retry_and_error_handler( - db, - user_db, - authed, - workspace_id, - script_path, - args, - retry, - error_handler_path, - error_handler_args, - trigger_path, - ) - .await - } -} - -async fn trigger_script_with_retry_and_error_handler( - db: &DB, - user_db: UserDB, - authed: ApiAuthed, - workspace_id: &str, - script_path: &str, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>>, - trigger_path: String, -) -> Result<(Uuid, Option)> { - #[cfg(feature = "enterprise")] - check_license_key_valid().await?; - - check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; - - let retry = retry.map(|r| r.0.clone()); - let error_handler_path = error_handler_path.map(|p| p.to_string()); - let error_handler_args = error_handler_args.map(|args| args.0.clone()); - - let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = { - let db_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; - script_path_to_payload( - script_path, - Some(db_authed), - db.clone(), - &workspace_id, - Some(false), - ) - .await? - }; - - check_tag_available_for_workspace(&db, &workspace_id, &tag, &authed).await?; - - let (email, permissioned_as, push_authed, tx) = - if let Some(on_behalf_of) = on_behalf_of.as_ref() { - ( - on_behalf_of.email.as_str(), - on_behalf_of.permissioned_as.clone(), - None, - PushIsolationLevel::IsolatedRoot(db.clone()), - ) - } else { - ( - authed.email.as_str(), - username_to_permissioned_as(&authed.username), - Some(authed.clone().into()), - PushIsolationLevel::Isolated(user_db, authed.clone().into()), - ) - }; - - let push_args = PushArgs { args: &args.args, extra: args.extra }; - - let retryable_job_payload = match job_payload { - JobPayload::ScriptHash { - hash, - path, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - cache_ttl, - priority, - apply_preprocessor, - .. - } => JobPayload::SingleScriptFlow { - path, - hash, - args: HashMap::from(&push_args), - retry, - error_handler_path, - error_handler_args, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - cache_ttl, - priority, - tag_override: tag.clone(), - apply_preprocessor, - trigger_path: Some(trigger_path), - }, - _ => { - return Err(windmill_common::error::Error::internal_err(format!( - "Unsupported job payload: {:?}", - job_payload - ))) - } - }; - - let (uuid, tx) = push( - &db, - tx, - &workspace_id, - retryable_job_payload, - push_args, - authed.display_username(), - email, - permissioned_as, - authed.token_prefix.as_deref(), - None, - None, - None, - None, - None, - None, - false, - false, - None, - true, - tag, - timeout, - None, - None, - push_authed.as_ref(), - false, - ) - .await?; - tx.commit().await?; - - Ok((uuid, delete_after_use)) -} diff --git a/backend/windmill-api/src/triggers/gcp/handler_oss.rs b/backend/windmill-api/src/triggers/gcp/handler_oss.rs index 825c7f71e0..4b52ff9c68 100644 --- a/backend/windmill-api/src/triggers/gcp/handler_oss.rs +++ b/backend/windmill-api/src/triggers/gcp/handler_oss.rs @@ -1,3 +1,4 @@ +#[allow(unused)] #[cfg(feature = "private")] pub use super::handler_ee::*; @@ -29,7 +30,7 @@ impl TriggerCrud for GcpTrigger { const SUPPORTS_TEST_CONNECTION: bool = false; const ROUTE_PREFIX: &'static str = "/gcp_triggers"; const DEPLOYMENT_NAME: &'static str = ""; - const IS_CLOUD_HOSTED: bool = false; + const IS_ALLOWED_ON_CLOUD: bool = false; fn get_deployed_object(path: String) -> DeployedObject { DeployedObject::GcpTrigger { path } diff --git a/backend/windmill-api/src/triggers/gcp/listener_oss.rs b/backend/windmill-api/src/triggers/gcp/listener_oss.rs new file mode 100644 index 0000000000..a14168968a --- /dev/null +++ b/backend/windmill-api/src/triggers/gcp/listener_oss.rs @@ -0,0 +1,41 @@ +#[allow(unused)] + +#[cfg(feature = "private")] +pub use super::listener_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::GcpTrigger, + crate::triggers::{listener::ListeningTrigger, Listener}, + std::sync::Arc, + tokio::sync::RwLock, + windmill_common::{error::Result, jobs::JobTriggerKind, DB}, +}; + +#[cfg(not(feature = "private"))] +#[async_trait::async_trait] +impl Listener for GcpTrigger { + type Consumer = (); + type Extra = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Gcp; + + async fn get_consumer( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + Ok(None) + } + async fn consume( + &self, + _db: &DB, + _consumer: Self::Consumer, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) { + () + } +} diff --git a/backend/windmill-api/src/triggers/gcp/mod.rs b/backend/windmill-api/src/triggers/gcp/mod.rs index f11ba94883..dd2f2a65df 100644 --- a/backend/windmill-api/src/triggers/gcp/mod.rs +++ b/backend/windmill-api/src/triggers/gcp/mod.rs @@ -2,6 +2,11 @@ mod handler_ee; pub mod handler_oss; + +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + #[cfg(feature = "private")] mod mod_ee; #[cfg(feature = "private")] diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index e2e5546e09..5d8898f121 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -1,10 +1,11 @@ use super::{ - http_trigger_args::RawHttpTriggerArgs, AuthenticationMethod, HttpMethod, TriggerRoute, - HTTP_ACCESS_CACHE, HTTP_AUTH_CACHE, HTTP_ROUTERS_CACHE, + http_trigger_args::RawHttpTriggerArgs, AuthenticationMethod, HttpMethod, RequestType, + TriggerRoute, HTTP_ACCESS_CACHE, HTTP_AUTH_CACHE, HTTP_ROUTERS_CACHE, }; use crate::{ auth::{AuthCache, OptTokened}, db::{ApiAuthed, DB}, + jobs::start_job_update_sse_stream, resources::try_get_resource_from_db_as, triggers::{ http::{ @@ -12,7 +13,8 @@ use crate::{ RouteExists, ROUTE_PATH_KEY_RE, VALID_ROUTE_PATH_RE, }, trigger_helpers::{ - get_runnable_format, trigger_runnable, trigger_runnable_and_wait_for_result, RunnableId, + get_runnable_format, trigger_runnable, trigger_runnable_and_wait_for_result, + trigger_runnable_inner, RunnableId, }, Trigger, TriggerCrud, TriggerData, }, @@ -26,6 +28,7 @@ use axum::{ routing::{get, post}, Extension, Json, Router, }; +use futures::StreamExt; use http::{HeaderMap, StatusCode}; use sqlx::PgConnection; use std::{ @@ -33,7 +36,6 @@ use std::{ collections::{HashMap, HashSet}, sync::Arc, }; -use tower_http::cors::CorsLayer; use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ db::UserDB, @@ -60,7 +62,9 @@ pub async fn increase_trigger_version(tx: &mut PgConnection) -> Result<()> { } pub fn generate_route_path_key(route_path: &str) -> String { - ROUTE_PATH_KEY_RE.replace_all(route_path, "/*").to_string() + ROUTE_PATH_KEY_RE + .replace_all(route_path, "${1}${2}key") + .to_string() } pub async fn route_path_key_exists( @@ -175,33 +179,35 @@ pub async fn insert_new_trigger_into_db( ) -> Result<()> { require_admin(authed.is_admin, &authed.username)?; + let request_type = trigger.config.request_type; + sqlx::query!( r#" INSERT INTO http_trigger ( - workspace_id, - path, - route_path, + workspace_id, + path, + route_path, route_path_key, workspaced_route, authentication_resource_path, wrap_body, raw_string, - script_path, + script_path, summary, description, - is_flow, - is_async, - authentication_method, - http_method, - static_asset_config, - edited_by, - email, - edited_at, + is_flow, + request_type, + authentication_method, + http_method, + static_asset_config, + edited_by, + email, + edited_at, is_static_website, error_handler_path, error_handler_args, retry - ) + ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, now(), $19, $20, $21, $22 ) @@ -218,7 +224,7 @@ pub async fn insert_new_trigger_into_db( trigger.config.summary, trigger.config.description, trigger.base.is_flow, - trigger.config.is_async, + request_type as _, trigger.config.authentication_method as _, trigger.config.http_method as _, trigger.config.static_asset_config as _, @@ -318,7 +324,7 @@ async fn check_if_route_exist( workspace_id: &str, trigger_path: Option<&str>, ) -> Result { - let route_path_key = ROUTE_PATH_KEY_RE.replace_all(&config.route_path, ":key"); + let route_path_key = generate_route_path_key(&config.route_path); let exists = route_path_key_exists( &route_path_key, @@ -336,7 +342,7 @@ async fn check_if_route_exist( )); } - Ok(route_path_key.into_owned()) + Ok(route_path_key) } pub struct HttpTrigger; @@ -359,7 +365,7 @@ impl TriggerCrud for HttpTrigger { const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ "route_path", "route_path_key", - "is_async", + "request_type", "authentication_method", "http_method", "summary", @@ -460,35 +466,37 @@ impl TriggerCrud for HttpTrigger { let route_path_key = check_if_route_exist(db, &trigger.config, workspace_id, Some(path)).await?; + let request_type = trigger.config.request_type; + sqlx::query!( r#" - UPDATE - http_trigger - SET - route_path = $1, + UPDATE + http_trigger + SET + route_path = $1, route_path_key = $2, workspaced_route = $3, wrap_body = $4, raw_string = $5, authentication_resource_path = $6, - script_path = $7, - path = $8, - is_flow = $9, - http_method = $10, - static_asset_config = $11, - edited_by = $12, - email = $13, - is_async = $14, - authentication_method = $15, + script_path = $7, + path = $8, + is_flow = $9, + http_method = $10, + static_asset_config = $11, + edited_by = $12, + email = $13, + request_type = $14, + authentication_method = $15, summary = $16, description = $17, - edited_at = now(), + edited_at = now(), is_static_website = $18, error_handler_path = $19, error_handler_args = $20, retry = $21 - WHERE - workspace_id = $22 AND + WHERE + workspace_id = $22 AND path = $23 "#, route_path, @@ -504,7 +512,7 @@ impl TriggerCrud for HttpTrigger { trigger.config.static_asset_config as _, &authed.username, &authed.email, - trigger.config.is_async, + request_type as _, trigger.config.authentication_method as _, trigger.config.summary, trigger.config.description, @@ -518,32 +526,34 @@ impl TriggerCrud for HttpTrigger { .execute(&mut *tx) .await?; } else { + let request_type = trigger.config.request_type; + sqlx::query!( r#" - UPDATE - http_trigger - SET + UPDATE + http_trigger + SET wrap_body = $1, raw_string = $2, authentication_resource_path = $3, - script_path = $4, - path = $5, - is_flow = $6, - http_method = $7, - static_asset_config = $8, - edited_by = $9, - email = $10, - is_async = $11, - authentication_method = $12, + script_path = $4, + path = $5, + is_flow = $6, + http_method = $7, + static_asset_config = $8, + edited_by = $9, + email = $10, + request_type = $11, + authentication_method = $12, summary = $13, description = $14, - edited_at = now(), + edited_at = now(), is_static_website = $15, error_handler_path = $16, error_handler_args = $17, retry = $18 - WHERE - workspace_id = $19 AND + WHERE + workspace_id = $19 AND path = $20 "#, trigger.config.wrap_body, @@ -556,7 +566,7 @@ impl TriggerCrud for HttpTrigger { trigger.config.static_asset_config as _, &authed.username, &authed.email, - trigger.config.is_async, + request_type as _, trigger.config.authentication_method as _, trigger.config.summary, trigger.config.description, @@ -598,17 +608,62 @@ impl TriggerCrud for HttpTrigger { } } +async fn conditional_cors_middleware( + req: axum::extract::Request, + next: axum::middleware::Next, +) -> Response { + let mut response = next.run(req).await; + + let headers = response.headers_mut(); + + // Check existing headers first to determine what not to insert + let mut not_insert_origin = false; + let mut not_insert_methods = false; + let mut not_insert_headers = false; + + for key in headers.keys() { + if !not_insert_origin && key == http::header::ACCESS_CONTROL_ALLOW_ORIGIN { + not_insert_origin = true; + } + if !not_insert_methods && key == http::header::ACCESS_CONTROL_ALLOW_METHODS { + not_insert_methods = true; + } + if !not_insert_headers && key == http::header::ACCESS_CONTROL_ALLOW_HEADERS { + not_insert_headers = true; + } + + // Early exit if all headers are already present + if not_insert_origin && not_insert_methods && not_insert_headers { + break; + } + } + + // Insert only the missing headers + if !not_insert_origin { + headers.insert( + http::header::ACCESS_CONTROL_ALLOW_ORIGIN, + http::HeaderValue::from_static("*"), + ); + } + + if !not_insert_methods { + headers.insert( + http::header::ACCESS_CONTROL_ALLOW_METHODS, + http::HeaderValue::from_static("GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS"), + ); + } + + if !not_insert_headers { + headers.insert( + http::header::ACCESS_CONTROL_ALLOW_HEADERS, + http::HeaderValue::from_static("content-type, authorization"), + ); + } + + response +} + pub fn http_route_trigger_handler() -> Router { - let cors = CorsLayer::new() - .allow_methods([ - http::Method::GET, - http::Method::POST, - http::Method::DELETE, - http::Method::PUT, - http::Method::PATCH, - ]) - .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION]) - .allow_origin(tower_http::cors::Any); Router::new() .route( "/*path", @@ -617,9 +672,10 @@ pub fn http_route_trigger_handler() -> Router { .delete(route_job) .put(route_job) .patch(route_job) - .head(|| async { "" }), + .head(|| async { "" }) + .options(|| async { "" }), ) - .layer(cors) + .layer(axum::middleware::from_fn(conditional_cors_middleware)) } async fn get_http_route_trigger( @@ -748,6 +804,7 @@ async fn route_job( args: RawHttpTriggerArgs, ) -> std::result::Result { let route_path = route_path.to_path().trim_end_matches("/"); + let (trigger, called_path, params, authed) = get_http_route_trigger( route_path, &auth_cache, @@ -976,8 +1033,85 @@ async fn route_job( ) .map_err(|e| e.into_response())?; - if trigger.is_async { - trigger_runnable( + // Handle execution based on the execution mode + match trigger.request_type { + RequestType::SyncSse => { + // Trigger the job (always async when streaming) + let (uuid, _, _) = trigger_runnable_inner( + &db, + Some(user_db.clone()), + authed.clone(), + &trigger.workspace_id, + &trigger.script_path, + trigger.is_flow, + args, + trigger.retry.as_ref(), + trigger.error_handler_path.as_deref(), + trigger.error_handler_args.as_ref(), + format!("http_trigger/{}", trigger.path), + None, + ) + .await + .map_err(|e| e.into_response())?; + + // Set up SSE stream + let opt_authed = Some(authed.clone()); + let opt_tokened = OptTokened { token: None }; + let (tx, rx) = tokio::sync::mpsc::channel(32); + + let stream = tokio_stream::wrappers::ReceiverStream::new(rx).map(|x| { + format!( + "data: {}\n\n", + serde_json::to_string(&x).unwrap_or_default() + ) + }); + + start_job_update_sse_stream( + opt_authed, + opt_tokened, + db.clone(), + trigger.workspace_id.clone(), + uuid, + None, + None, + None, + None, + Some(true), + Some(true), + None, + None, + tx, + None, + ); + + let body = axum::body::Body::from_stream( + stream.map(std::result::Result::<_, std::convert::Infallible>::Ok), + ); + + Ok(Response::builder() + .status(200) + .header("Content-Type", "text/event-stream") + .header("Cache-Control", "no-cache") + .body(body) + .map_err(|e| Error::internal_err(e.to_string()).into_response())?) + } + RequestType::Async => trigger_runnable( + &db, + Some(user_db), + authed, + &trigger.workspace_id, + &trigger.script_path, + trigger.is_flow, + args, + trigger.retry.as_ref(), + trigger.error_handler_path.as_deref(), + trigger.error_handler_args.as_ref(), + format!("http_trigger/{}", trigger.path), + None, + ) + .await + .map_err(|e| e.into_response()), + RequestType::Sync => trigger_runnable_and_wait_for_result( &db, Some(user_db), authed, @@ -991,22 +1125,6 @@ async fn route_job( format!("http_trigger/{}", trigger.path), ) .await - .map_err(|e| e.into_response()) - } else { - trigger_runnable_and_wait_for_result( - &db, - Some(user_db), - authed, - &trigger.workspace_id, - &trigger.script_path, - trigger.is_flow, - args, - trigger.retry.as_ref(), - trigger.error_handler_path.as_deref(), - trigger.error_handler_args.as_ref(), - format!("http_trigger/{}", trigger.path), - ) - .await - .map_err(|e| e.into_response()) + .map_err(|e| e.into_response()), } } diff --git a/backend/windmill-api/src/triggers/http/http_trigger_args.rs b/backend/windmill-api/src/triggers/http/http_trigger_args.rs index 8fb96d42f0..af829cec53 100644 --- a/backend/windmill-api/src/triggers/http/http_trigger_args.rs +++ b/backend/windmill-api/src/triggers/http/http_trigger_args.rs @@ -15,7 +15,10 @@ use windmill_common::{ use windmill_queue::PushArgsOwned; use crate::{ - args::{try_from_request_body, Body, RawWebhookArgs, WebhookArgs, WebhookArgsMetadata}, + args::{ + build_headers, build_query, try_from_request_body, Body, RawWebhookArgs, WebhookArgs, + WebhookArgsMetadata, + }, db::ApiAuthed, }; @@ -102,8 +105,8 @@ struct HttpTriggerWmTrigger<'a> { route: &'a str, path: &'a str, params: &'a HashMap, - query: &'a HashMap>, - headers: &'a HashMap>, + query: HashMap>, + headers: HashMap>, method: HttpMethod, } @@ -142,14 +145,22 @@ impl HttpTriggerArgs { format: RunnableFormat, wrap_body: bool, ) -> Result { + let headers = build_headers(&self.0.metadata.headers, None, true); + let query = build_query(self.0.metadata.query.as_deref(), None, true); match format { RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V2 } => { // we don't care about wrap_body in v2 - self.to_v2_preprocessor_args(route_path, called_path, params) - } - RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V1 } => { - self.to_v1_preprocessor_args(route_path, called_path, params, wrap_body) + self.to_v2_preprocessor_args(route_path, called_path, params, headers, query) } + RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V1 } => self + .to_v1_preprocessor_args( + route_path, + called_path, + params, + wrap_body, + headers, + query, + ), RunnableFormat { has_preprocessor: false, .. } => self.to_main_args(wrap_body), } } @@ -160,6 +171,8 @@ impl HttpTriggerArgs { called_path: &str, params: &HashMap, wrap_body: bool, + headers: HashMap>, + query: HashMap>, ) -> Result { let mut extra = HashMap::new(); let mut wm_trigger = HashMap::new(); @@ -171,8 +184,8 @@ impl HttpTriggerArgs { path: called_path, method: (&self.0.metadata.method).try_into()?, params, - query: &self.0.metadata.query, - headers: &self.0.metadata.headers, + query, + headers, }), ); extra.insert("wm_trigger".to_string(), to_raw_value(&wm_trigger)); @@ -189,6 +202,8 @@ impl HttpTriggerArgs { route_path: &str, called_path: &str, params: &HashMap, + headers: HashMap>, + query: HashMap>, ) -> Result { let mut args = HashMap::new(); args.insert( @@ -197,8 +212,8 @@ impl HttpTriggerArgs { kind: "http".to_string(), body: to_raw_value(&self.0.body), raw_string: self.0.metadata.raw_string, - headers: self.0.metadata.headers, - query: self.0.metadata.query, + headers, + query, method: (&self.0.metadata.method).try_into()?, route: route_path, path: called_path, diff --git a/backend/windmill-api/src/triggers/http/mod.rs b/backend/windmill-api/src/triggers/http/mod.rs index 2b29f0e644..55844e0fb1 100644 --- a/backend/windmill-api/src/triggers/http/mod.rs +++ b/backend/windmill-api/src/triggers/http/mod.rs @@ -35,7 +35,7 @@ pub struct TriggerRoute { is_flow: bool, route_path: String, workspace_id: String, - is_async: bool, + request_type: RequestType, authentication_method: AuthenticationMethod, edited_by: String, email: String, @@ -66,6 +66,15 @@ pub enum HttpMethod { Patch, } +#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, PartialEq)] +#[sqlx(type_name = "REQUEST_TYPE", rename_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum RequestType { + Sync, + Async, + SyncSse, +} + impl TryFrom<&http::Method> for HttpMethod { type Error = Error; fn try_from(method: &http::Method) -> Result { @@ -96,7 +105,7 @@ pub enum AuthenticationMethod { pub struct HttpConfig { pub route_path: String, pub route_path_key: String, - pub is_async: bool, + pub request_type: RequestType, pub authentication_method: AuthenticationMethod, pub http_method: HttpMethod, pub summary: Option, @@ -109,11 +118,11 @@ pub struct HttpConfig { pub raw_string: bool, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize)] pub struct HttpConfigRequest { #[serde(default)] pub route_path: String, - pub is_async: bool, + pub request_type: RequestType, pub authentication_method: AuthenticationMethod, pub http_method: HttpMethod, pub summary: Option, @@ -126,10 +135,66 @@ pub struct HttpConfigRequest { pub raw_string: Option, } +#[derive(Deserialize)] +struct HttpConfigRequestHelper { + #[serde(default)] + route_path: String, + request_type: Option, + is_async: Option, + authentication_method: AuthenticationMethod, + http_method: HttpMethod, + summary: Option, + description: Option, + static_asset_config: Option>, + is_static_website: bool, + authentication_resource_path: Option, + workspaced_route: Option, + wrap_body: Option, + raw_string: Option, +} + +impl<'de> Deserialize<'de> for HttpConfigRequest { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + let helper = HttpConfigRequestHelper::deserialize(deserializer)?; + + // Determine request_type with backward compatibility + let request_type = if let Some(mode) = helper.request_type { + mode + } else if let Some(is_async) = helper.is_async { + if is_async { + RequestType::Async + } else { + RequestType::Sync + } + } else { + RequestType::Sync + }; + + Ok(HttpConfigRequest { + route_path: helper.route_path, + request_type, + authentication_method: helper.authentication_method, + http_method: helper.http_method, + summary: helper.summary, + description: helper.description, + static_asset_config: helper.static_asset_config, + is_static_website: helper.is_static_website, + authentication_resource_path: helper.authentication_resource_path, + workspaced_route: helper.workspaced_route, + wrap_body: helper.wrap_body, + raw_string: helper.raw_string, + }) + } +} + // Regex patterns for route validation lazy_static::lazy_static! { - static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/?:[-\w]+").unwrap(); - static ref VALID_ROUTE_PATH_RE: regex::Regex = regex::Regex::new(r"^:?[-\w]+(/:?[-\w]+)*$").unwrap(); + // Matches named params like :id or wildcards like :* or * + static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"(/)?(:|\*)[-\w]+").unwrap(); + static ref VALID_ROUTE_PATH_RE: regex::Regex = regex::Regex::new(r"^(\*[-\w]+$|:?[-\w]+)(/(\*[-\w]+$|:?[-\w]+))*$").unwrap(); } #[derive(Deserialize)] @@ -174,17 +239,17 @@ pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, Route let triggers = sqlx::query_as!( TriggerRoute, r#" - SELECT - path, - script_path, - is_flow, - route_path, + SELECT + path, + script_path, + is_flow, + route_path, authentication_resource_path, - workspace_id, - is_async, - authentication_method AS "authentication_method: _", - edited_by, - email, + workspace_id, + request_type AS "request_type: _", + authentication_method AS "authentication_method: _", + edited_by, + email, static_asset_config AS "static_asset_config: _", wrap_body, raw_string, @@ -193,9 +258,9 @@ pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, Route error_handler_path, error_handler_args as "error_handler_args: _", retry as "retry: _" - FROM - http_trigger - WHERE + FROM + http_trigger + WHERE http_method = $1 "#, &http_method as &HttpMethod @@ -277,3 +342,66 @@ pub async fn refresh_routers_loop( } }); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_request_type_backward_compatibility() { + // Test with new request_type field + let json_new = r#"{ + "route_path": "/test", + "request_type": "sync_sse", + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_new).unwrap(); + assert_eq!(config.request_type, RequestType::SyncSse); + + // Test with legacy is_async = true + let json_legacy_async = r#"{ + "route_path": "/test", + "is_async": true, + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_legacy_async).unwrap(); + assert_eq!(config.request_type, RequestType::Async); + + // Test with legacy is_async = false + let json_legacy_sync = r#"{ + "route_path": "/test", + "is_async": false, + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_legacy_sync).unwrap(); + assert_eq!(config.request_type, RequestType::Sync); + + // Test with neither field (default to sync) + let json_default = r#"{ + "route_path": "/test", + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_default).unwrap(); + assert_eq!(config.request_type, RequestType::Sync); + + // Test that request_type takes precedence over is_async + let json_both = r#"{ + "route_path": "/test", + "request_type": "sync_sse", + "is_async": true, + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_both).unwrap(); + assert_eq!(config.request_type, RequestType::SyncSse); + } +} diff --git a/backend/windmill-api/src/triggers/kafka/handler_oss.rs b/backend/windmill-api/src/triggers/kafka/handler_oss.rs index 3922b684ce..bf5537b941 100644 --- a/backend/windmill-api/src/triggers/kafka/handler_oss.rs +++ b/backend/windmill-api/src/triggers/kafka/handler_oss.rs @@ -33,7 +33,7 @@ impl TriggerCrud for KafkaTrigger { const SUPPORTS_TEST_CONNECTION: bool = false; const ROUTE_PREFIX: &'static str = "/kafka_triggers"; const DEPLOYMENT_NAME: &'static str = ""; - const IS_CLOUD_HOSTED: bool = false; + const IS_ALLOWED_ON_CLOUD: bool = false; fn get_deployed_object(path: String) -> DeployedObject { DeployedObject::KafkaTrigger { path } diff --git a/backend/windmill-api/src/triggers/kafka/listener_oss.rs b/backend/windmill-api/src/triggers/kafka/listener_oss.rs new file mode 100644 index 0000000000..c5dca4ab37 --- /dev/null +++ b/backend/windmill-api/src/triggers/kafka/listener_oss.rs @@ -0,0 +1,41 @@ +#[allow(unused)] + +#[cfg(feature = "private")] +pub use super::listener_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::KafkaTrigger, + crate::triggers::{listener::ListeningTrigger, Listener}, + std::sync::Arc, + tokio::sync::RwLock, + windmill_common::{error::Result, jobs::JobTriggerKind, DB}, +}; + +#[cfg(not(feature = "private"))] +#[async_trait::async_trait] +impl Listener for KafkaTrigger { + type Consumer = (); + type Extra = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Kafka; + + async fn get_consumer( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + Ok(None) + } + async fn consume( + &self, + _db: &DB, + _consumer: Self::Consumer, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) { + () + } +} diff --git a/backend/windmill-api/src/triggers/kafka/mod.rs b/backend/windmill-api/src/triggers/kafka/mod.rs index 700b76b056..f09b2283ff 100644 --- a/backend/windmill-api/src/triggers/kafka/mod.rs +++ b/backend/windmill-api/src/triggers/kafka/mod.rs @@ -2,6 +2,10 @@ mod handler_ee; pub mod handler_oss; +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + #[cfg(feature = "private")] mod mod_ee; #[cfg(feature = "private")] diff --git a/backend/windmill-api/src/triggers/listener.rs b/backend/windmill-api/src/triggers/listener.rs new file mode 100644 index 0000000000..431c4313f9 --- /dev/null +++ b/backend/windmill-api/src/triggers/listener.rs @@ -0,0 +1,916 @@ +use std::{collections::HashMap, fmt::Debug, sync::Arc}; + +use crate::{ + capture::insert_capture_payload, + db::ApiAuthed, + triggers::{ + handler::TriggerCrud, + trigger_helpers::{trigger_runnable, TriggerJobArgs}, + Trigger, TriggerErrorHandling, + }, + users::fetch_api_authed, +}; +use async_trait::async_trait; +use itertools::Itertools; +use rand::seq::SliceRandom; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sql_builder::SqlBuilder; +use sqlx::{FromRow, Row}; +use tokio::sync::RwLock; +use windmill_common::{ + error::{Error, Result}, + jobs::JobTriggerKind, + triggers::TriggerKind, + utils::report_critical_error, + DB, INSTANCE_NAME, +}; + +#[allow(unused)] +#[async_trait] +pub trait Listener: TriggerCrud + TriggerJobArgs { + type Consumer: Send; + type Extra: Send + Sync; + type ExtraState: Send + Sync; + + //to use in next PR to add job trigger kind to eow + #[allow(unused)] + const JOB_TRIGGER_KIND: JobTriggerKind; + const EXTRA_TRIGGER_AND_WHERE_CLAUSE: &[&'static str] = &[]; + const EXTRA_CAPTURE_AND_WHERE_CLAUSE: &[&'static str] = &[]; + + async fn get_consumer( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result>; + async fn consume( + &self, + db: &DB, + consumer: Self::Consumer, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + killpill_rx: tokio::sync::broadcast::Receiver<()>, + extra: Option<&Self::ExtraState>, + ); + async fn fetch_enabled_unlistened_triggers( + &self, + db: &DB, + ) -> Result>> { + let mut fields = vec![ + "workspace_id", + "path", + "script_path", + "is_flow", + "edited_by", + "email", + "edited_at", + "extra_perms", + ]; + + if Self::SUPPORTS_SERVER_STATE { + fields.extend_from_slice(&["enabled", "server_id", "last_server_ping", "error"]); + } + fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); + fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); + + let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); + + sqlb.fields(&fields).and_where("enabled IS TRUE").and_where( + "(last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", + ); + + for where_clause in Self::EXTRA_TRIGGER_AND_WHERE_CLAUSE { + sqlb.and_where(where_clause); + } + + let sql = sqlb + .sql() + .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; + + let triggers: Vec> = + sqlx::query_as(&sql).fetch_all(db).await?; + + let triggers = triggers + .into_iter() + .map(|trigger| ListeningTrigger { + path: trigger.base.path, + workspace_id: trigger.base.workspace_id, + is_flow: trigger.base.is_flow, + username: trigger.base.edited_by, + email: trigger.base.email, + script_path: trigger.base.script_path, + trigger_config: trigger.config, + error_handling: Some(trigger.error_handling), + trigger_mode: true, + }) + .collect_vec(); + + Ok(triggers) + } + + async fn fetch_unlistened_captures( + &self, + db: &DB, + ) -> Result>> { + let fields = vec![ + "path", + "is_flow", + "workspace_id", + "owner AS username", + "email", + "trigger_config", + ]; + + let mut sqlb = SqlBuilder::select_from("capture_config"); + sqlb.fields(&fields) + .and_where(format!("trigger_kind = '{}'", Self::TRIGGER_KIND.to_key())) + .and_where("last_client_ping > NOW() - INTERVAL '10 seconds'") + .and_where("trigger_config IS NOT NULL") + .and_where( + "(last_server_ping IS NULL OR last_server_ping < NOW() - INTERVAL '15 seconds')", + ); + + for where_clause in Self::EXTRA_CAPTURE_AND_WHERE_CLAUSE { + sqlb.and_where(where_clause); + } + + let sql = sqlb.sql().expect("failed to build SQL"); + + let captures: Vec> = + sqlx::query_as(&sql).fetch_all(db).await?; + + let captures = captures + .into_iter() + .map(|capture| ListeningTrigger { + username: capture.username, + path: capture.path, + workspace_id: capture.workspace_id, + script_path: "".to_string(), + email: capture.email, + trigger_config: capture.trigger_config, + trigger_mode: false, + is_flow: capture.is_flow, + error_handling: None, + }) + .collect_vec(); + + Ok(captures) + } + + async fn get_extra_state(&self) -> Option { + None + } + + async fn cleanup( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _extra: Option<&Self::ExtraState>, + ) -> Result<()> { + Ok(()) + } + + async fn loop_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + status: Arc>>, + error_message: Option, + ) { + update_rw_lock(status.clone(), error_message).await; + loop { + if let None = self + .update_ping(db, listening_trigger, status.read().await.as_deref()) + .await + { + return; + } + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + } + } + + async fn update_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: Option<&str>, + ) -> Option<()> { + if listening_trigger.trigger_mode { + self.update_trigger_ping(db, listening_trigger, error).await + } else { + self.update_capture_ping(db, listening_trigger, error).await + } + } + + async fn update_ping_and_loop_ping_status( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + loop_ping_status: Arc>>, + error: Option, + ) -> Option<()> { + // update immediately the ping status and update the loop ping status so that the next loop pings will display the new status + update_rw_lock(loop_ping_status.clone(), error.clone()).await; + if let None = self + .update_ping(db, listening_trigger, error.as_deref()) + .await + { + return None; + } + Some(()) + } + + async fn update_trigger_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: Option<&str>, + ) -> Option<()> { + let updated = sqlx::query_scalar::<_, i32>(&format!( + r#" + UPDATE + {} + SET + last_server_ping = now(), error = $1 + WHERE + workspace_id = $2 AND + path = $3 AND + server_id = $4 AND + enabled IS TRUE + RETURNING 1 + "#, + Self::TABLE_NAME + )) + .bind(error) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .bind(&*INSTANCE_NAME) + .fetch_optional(db) + .await; + + self.handle_ping_result(updated, db, listening_trigger, "trigger") + .await + } + + async fn update_capture_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: Option<&str>, + ) -> Option<()> { + let updated = sqlx::query_scalar!( + r#" + UPDATE + capture_config + SET + last_server_ping = now(), error = $1 + WHERE + workspace_id = $2 AND + path = $3 AND + is_flow = $4 AND + trigger_kind = $5 AND + server_id = $6 AND + last_client_ping > NOW() - INTERVAL '10 seconds' + RETURNING 1 + "#, + error, + &listening_trigger.workspace_id, + &listening_trigger.path, + &listening_trigger.is_flow, + Self::TRIGGER_KIND as TriggerKind, + &*INSTANCE_NAME + ) + .fetch_optional(db) + .await + .map(|result| result.flatten()); + + self.handle_ping_result(updated, db, listening_trigger, "capture") + .await + } + + async fn handle_ping_result( + &self, + result: sqlx::Result>, + db: &DB, + listening_trigger: &ListeningTrigger, + entity_type: &str, + ) -> Option<()> { + match result { + Ok(updated) => { + if updated.is_none() { + self.reset_ping_for_restart(db, listening_trigger).await; + tracing::info!( + "{} {} {} changed, disabled, or deleted, stopping...", + Self::TRIGGER_KIND, + entity_type, + listening_trigger.path + ); + return None; + } + } + Err(error) => { + tracing::warn!( + "Error updating ping of {} {} {}: {:?}", + Self::TRIGGER_KIND, + entity_type, + &listening_trigger.path, + error + ); + } + } + + Some(()) + } + + async fn reset_ping_for_restart( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + ) { + if listening_trigger.trigger_mode { + let _ = sqlx::query(&format!( + r#" + UPDATE + {} + SET + last_server_ping = NULL + WHERE + workspace_id = $1 AND + path = $2 AND + server_id IS NULL + "#, + Self::TABLE_NAME + )) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .execute(db) + .await; + } else { + let _ = sqlx::query!( + r#" + UPDATE + capture_config + SET + last_server_ping = NULL + WHERE + workspace_id = $1 AND + path = $2 AND + is_flow = $3 AND + trigger_kind = $4 AND + server_id IS NULL + "#, + &listening_trigger.workspace_id, + &listening_trigger.path, + &listening_trigger.is_flow, + Self::TRIGGER_KIND as TriggerKind + ) + .execute(db) + .await; + } + } + + async fn disable_with_error( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: String, + ) { + if listening_trigger.trigger_mode { + let report_status = sqlx::query(&format!( + r#" + UPDATE + {} + SET + enabled = FALSE, + error = $1, + server_id = NULL, + last_server_ping = NULL + WHERE + workspace_id = $2 AND + path = $3 + "#, + Self::TABLE_NAME + )) + .bind(&error) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .execute(db) + .await; + + match report_status { + Ok(_) => { + report_critical_error( + format!( + "Disabling {} trigger {} because of error: {}", + Self::TRIGGER_KIND, + listening_trigger.path, + error + ), + db.clone(), + Some(&listening_trigger.workspace_id), + None, + ) + .await; + } + Err(disable_err) => { + report_critical_error( + format!("Could not disable {} trigger {} with err {}, disabling because of error {}", Self::TRIGGER_KIND, listening_trigger.path, disable_err, error), + db.clone(), + Some(&listening_trigger.workspace_id), + None, + ).await; + } + } + return; + } + + let report_status = sqlx::query!( + r#" + UPDATE + capture_config + SET + error = $1, + server_id = NULL, + last_server_ping = NULL + WHERE + workspace_id = $2 AND + path = $3 AND + is_flow = $4 AND + trigger_kind = $5 + "#, + error, + listening_trigger.workspace_id, + listening_trigger.path, + listening_trigger.is_flow, + Self::TRIGGER_KIND as TriggerKind + ) + .execute(db) + .await; + + if let Err(disable_err) = report_status { + tracing::error!( + "Could not disable {} capture {} ({}) with err {}, disabling because of error {}", + Self::TRIGGER_KIND, + listening_trigger.path, + listening_trigger.workspace_id, + disable_err, + error + ) + } + } + + async fn handle_trigger( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + payload: Self::Payload, + trigger_info: HashMap>, + _extra: Option, + ) -> Result<()> { + let args = Self::build_job_args( + &listening_trigger.script_path, + listening_trigger.is_flow, + &listening_trigger.workspace_id, + db, + payload, + trigger_info, + ) + .await?; + + let authed = listening_trigger + .authed(db, &Self::TRIGGER_KIND.to_string()) + .await?; + + let (retry, error_handler_path, error_handler_args) = + match listening_trigger.error_handling.as_ref() { + Some(error_handling) => ( + error_handling.retry.as_ref(), + error_handling.error_handler_path.as_deref(), + error_handling.error_handler_args.as_ref(), + ), + None => (None, None, None), + }; + + tracing::debug!( + "Triggering job from {} event {} with args {:?}", + Self::TRIGGER_KIND, + listening_trigger.path, + args + ); + + trigger_runnable( + db, + None, + authed, + &listening_trigger.workspace_id, + &listening_trigger.script_path, + listening_trigger.is_flow, + args, + retry, + error_handler_path.as_deref(), + error_handler_args, + format!("{}_trigger/{}", Self::TRIGGER_KIND, listening_trigger.path), + None, + ) + .await?; + + Ok(()) + } + + async fn handle_event( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + payload: Self::Payload, + trigger_info: HashMap>, + extra: Option, + ) -> Result<()> { + if listening_trigger.trigger_mode { + if let Err(err) = self + .handle_trigger(db, listening_trigger, payload, trigger_info, extra) + .await + { + report_critical_error( + format!( + "Failed to trigger job from {} event {}: {:?}", + Self::TRIGGER_KIND, + listening_trigger.path, + err + ), + db.clone(), + Some(&listening_trigger.workspace_id), + None, + ) + .await; + return Err(err); + }; + return Ok(()); + } + + let (main_args, preprocessor_args) = Self::build_capture_payloads(&payload, trigger_info); + if let Err(err) = insert_capture_payload( + db, + &listening_trigger.workspace_id, + &listening_trigger.path, + listening_trigger.is_flow, + &Self::TRIGGER_KIND, + main_args, + preprocessor_args, + &listening_trigger.username, + ) + .await + { + tracing::error!("Error inserting capture payload: {:?}", err); + return Err(err); + } + Ok(()) + } +} + +#[allow(unused)] +async fn listening( + db: DB, + listener: T, + listening_trigger: ListeningTrigger, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) { + let killpill_rx_consumer = killpill_rx.resubscribe(); + let killpill_rx_get_consumer = killpill_rx.resubscribe(); + + let loop_ping_status = Arc::new(RwLock::new(None)); + let extra_state = listener.get_extra_state().await; + tokio::select! { + biased; + _ = killpill_rx.recv() => { + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + } + _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), Some("Connecting...".to_string())) => { + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + } + consumer = { + listener.get_consumer(&db, &listening_trigger, loop_ping_status.clone(), killpill_rx_get_consumer) + } => { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + return; + } + _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), None) => { + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + return; + } + _ = async { + match consumer { + Ok(Some(consumer)) => { + listener.update_ping_and_loop_ping_status(&db, &listening_trigger, loop_ping_status.clone(), None).await; + let _ = listener.consume(&db, consumer, &listening_trigger, loop_ping_status.clone(), killpill_rx_consumer, extra_state.as_ref()).await; + tracing::debug!("Stopping consumer for trigger"); + } + Err(error) => { + tracing::warn!("Disabling trigger due to consumer error: {}", error); + listener.disable_with_error(&db, &listening_trigger, error.to_string()).await; + } + _ => {} + } + } => { + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + return; + } + } + } + } +} + +#[allow(unused)] +async fn listen_to_unlistened_events( + listener: T, + db: DB, + killpill_rx: &tokio::sync::broadcast::Receiver<()>, +) { + let unlistend_enabled_triggers = listener.fetch_enabled_unlistened_triggers(&db).await; + + match unlistend_enabled_triggers { + Ok(mut unlistend_enabled_triggers) => { + unlistend_enabled_triggers.shuffle(&mut rand::rng()); + for trigger in unlistend_enabled_triggers { + let has_lock = sqlx::query_scalar(&format!( + r#" + UPDATE + {} + SET + server_id = $1, + last_server_ping = now(), + error = 'Connecting...' + WHERE + enabled IS TRUE + AND workspace_id = $2 + AND path = $3 + AND (last_server_ping IS NULL + OR last_server_ping < now() - INTERVAL '15 seconds' + ) + RETURNING true + "#, + T::TABLE_NAME, + )) + .bind(&*INSTANCE_NAME) + .bind(&trigger.workspace_id) + .bind(&trigger.path) + .fetch_optional(&db) + .await; + match has_lock { + Ok(has_lock) => { + if has_lock.flatten().unwrap_or(false) { + tracing::info!( + "Spawning new task to listen for {} event", + T::TABLE_NAME + ); + tokio::spawn({ + let db = db.clone(); + let killpill_rx = killpill_rx.resubscribe(); + async move { listening(db, listener, trigger, killpill_rx).await } + }); + } else { + tracing::info!( + "{} trigger {} already being listened to", + T::TRIGGER_KIND, + trigger.path + ); + } + } + Err(err) => { + tracing::error!( + "Error acquiring lock for {} trigger {}: {:?}", + T::TRIGGER_KIND, + trigger.path, + err + ); + } + }; + } + } + Err(err) => { + tracing::error!("Error fetching {} triggers: {:?}", T::TRIGGER_KIND, err,); + } + } + + let unlisted_captures = listener.fetch_unlistened_captures(&db).await; + + match unlisted_captures { + Ok(unlistened_captures) => { + for capture in unlistened_captures { + let has_lock = sqlx::query_scalar!( + r#" + UPDATE + capture_config + SET + server_id = $1, + last_server_ping = now(), + error = 'Connecting...' + WHERE + last_client_ping > NOW() - INTERVAL '10 seconds' AND + workspace_id = $2 AND + path = $3 AND + is_flow = $4 AND + trigger_kind = $5 AND + (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') + RETURNING true + "#, + *INSTANCE_NAME, + &capture.workspace_id, + &capture.path, + &capture.is_flow, + T::TRIGGER_KIND as TriggerKind + ) + .fetch_optional(&db) + .await; + match has_lock { + Ok(has_lock) => { + if has_lock.flatten().unwrap_or(false) { + tokio::spawn({ + let db = db.clone(); + let killpill_rx = killpill_rx.resubscribe(); + async move { listening(db, listener, capture, killpill_rx).await } + }); + } else { + tracing::info!( + "{} capture {} already being listened to", + T::TRIGGER_KIND.to_string(), + capture.path + ); + } + } + Err(err) => { + tracing::error!( + "Error acquiring lock for capture {} {}: {:?}", + T::TRIGGER_KIND, + capture.path, + err + ); + } + }; + } + } + Err(err) => { + tracing::error!( + "Error fetching captures {} triggers: {:?}", + T::TRIGGER_KIND, + err + ); + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +struct Capture +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, +{ + path: String, + is_flow: bool, + workspace_id: String, + username: String, + email: String, + #[serde(flatten)] + trigger_config: T, +} + +impl FromRow<'_, sqlx::postgres::PgRow> for Capture +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow> + DeserializeOwned, +{ + fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result { + let trigger_config_value = row.try_get("trigger_config")?; + let trigger_config: T = serde_json::from_value(trigger_config_value) + .map_err(|e| sqlx::Error::Decode(Box::new(e)))?; + Ok(Capture { + path: row.try_get("path")?, + is_flow: row.try_get("is_flow")?, + workspace_id: row.try_get("workspace_id")?, + username: row.try_get("username")?, + email: row.try_get("email")?, + trigger_config, + }) + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ListeningTrigger { + pub path: String, + pub is_flow: bool, + pub workspace_id: String, + pub username: String, + pub email: String, + pub trigger_config: T, + pub script_path: String, + pub trigger_mode: bool, + pub error_handling: Option, +} + +impl ListeningTrigger { + pub async fn authed(&self, db: &DB, username: &str) -> Result { + fetch_api_authed( + self.username.clone(), + self.email.clone(), + &self.workspace_id, + db, + Some(format!("{}-{}", username, self.path)), + ) + .await + } +} + +#[allow(unused)] +pub async fn update_rw_lock(lock: std::sync::Arc>, value: T) -> () { + let mut w = lock.write().await; + *w = value; +} + +#[allow(unused)] +fn listen_to( + trigger: T, + db: DB, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) { + tokio::spawn(async move { + listen_to_unlistened_events(trigger, db.clone(), &killpill_rx).await; + loop { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return; + } + _ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => { + listen_to_unlistened_events(trigger, db.clone(), &killpill_rx).await + } + } + } + }); +} + +#[allow(unused)] +pub fn start_all_listeners(db: DB, killpill_rx: &tokio::sync::broadcast::Receiver<()>) { + tracing::info!("Starting trigger listeners based on available features..."); + + #[cfg(feature = "postgres_trigger")] + { + let postgres_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::postgres::PostgresTrigger; + + listen_to(PostgresTrigger, db.clone(), postgres_killpill_rx) + } + + #[cfg(feature = "mqtt_trigger")] + { + let mqtt_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::mqtt::MqttTrigger; + + listen_to(MqttTrigger, db.clone(), mqtt_killpill_rx) + } + + #[cfg(feature = "websocket")] + { + let mqtt_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::websocket::WebsocketTrigger; + + listen_to(WebsocketTrigger, db.clone(), mqtt_killpill_rx) + } + + #[cfg(all(feature = "gcp_trigger", feature = "enterprise", feature = "private"))] + { + let gcp_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::gcp::GcpTrigger; + + listen_to(GcpTrigger, db.clone(), gcp_killpill_rx); + } + + #[cfg(all(feature = "sqs_trigger", feature = "enterprise", feature = "private"))] + { + let gcp_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::sqs::SqsTrigger; + + listen_to(SqsTrigger, db.clone(), gcp_killpill_rx); + } + + #[cfg(all(feature = "nats", feature = "enterprise", feature = "private"))] + { + let gcp_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::nats::NatsTrigger; + + listen_to(NatsTrigger, db.clone(), gcp_killpill_rx); + } + + #[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))] + { + let gcp_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::kafka::KafkaTrigger; + + listen_to(KafkaTrigger, db.clone(), gcp_killpill_rx); + } + + tracing::info!("All available trigger listeners have been started"); +} diff --git a/backend/windmill-api/src/triggers/mod.rs b/backend/windmill-api/src/triggers/mod.rs index a5e6dc61f2..9a7f22e3e5 100644 --- a/backend/windmill-api/src/triggers/mod.rs +++ b/backend/windmill-api/src/triggers/mod.rs @@ -23,11 +23,15 @@ pub mod sqs; pub mod websocket; mod handler; +mod listener; pub mod trigger_helpers; #[allow(unused)] pub(crate) use handler::TriggerCrud; pub use handler::{generate_trigger_routers, get_triggers_count_internal, TriggersCount}; +pub use listener::start_all_listeners; +#[allow(unused)] +pub(crate) use listener::Listener; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StandardTriggerQuery { diff --git a/backend/windmill-api/src/triggers/mqtt/listener.rs b/backend/windmill-api/src/triggers/mqtt/listener.rs new file mode 100644 index 0000000000..af2d94b113 --- /dev/null +++ b/backend/windmill-api/src/triggers/mqtt/listener.rs @@ -0,0 +1,376 @@ +use std::{collections::HashMap, sync::Arc}; + +use async_trait::async_trait; +use bytes::Bytes; +use rumqttc::{ + v5::{ + mqttbytes::v5::PublishProperties, Event as V5Event, EventLoop as V5EventLoop, + Incoming as V5Incoming, + }, + Event as V3Event, EventLoop as V3EventLoop, Incoming as V3Incoming, +}; +use std::time::Duration; +use tokio::sync::RwLock; +use windmill_common::{ + db::UserDB, + error::{to_anyhow, Error, Result}, + jobs::JobTriggerKind, + worker::to_raw_value, + DB, +}; + +use crate::{ + resources::try_get_resource_from_db_as, + triggers::{ + listener::ListeningTrigger, + mqtt::{ + MqttClientBuilder, MqttClientResult, MqttConfig, MqttResource, MqttTrigger, + V3MqttHandler, V5MqttHandler, + }, + trigger_helpers::TriggerJobArgs, + Listener, + }, +}; + +#[async_trait] +impl Listener for MqttTrigger { + type Consumer = MqttClientResult; + type Extra = (); + type ExtraState = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Mqtt; + + async fn get_consumer( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + let ListeningTrigger:: { workspace_id, trigger_config, .. } = + listening_trigger; + + let MqttConfig { + mqtt_resource_path, + subscribe_topics, + v3_config, + v5_config, + client_id, + client_version, + .. + } = trigger_config; + + let authed = listening_trigger + .authed(db, &Self::TRIGGER_KIND.to_string()) + .await?; + + let mqtt_resource = try_get_resource_from_db_as::( + &authed, + Some(UserDB::new(db.clone())), + &db, + mqtt_resource_path, + workspace_id, + ) + .await?; + + let subscribe_topics = subscribe_topics + .iter() + .map(|topic| topic.0.clone()) + .collect(); + + let client_builder = MqttClientBuilder::new( + mqtt_resource, + client_id.as_deref(), + subscribe_topics, + v3_config.as_ref().map(|c| &c.0), + v5_config.as_ref().map(|c| &c.0), + client_version.as_ref(), + ); + + let client_result = client_builder + .build_client() + .await + .map_err(|e| Error::BadConfig(format!("Failed to build MQTT client: {}", e)))?; + + Ok(Some(client_result)) + } + + async fn consume( + &self, + db: &DB, + consumer: Self::Consumer, + listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, + ) { + tracing::info!( + "Starting to listen for MQTT trigger {}", + &listening_trigger.path + ); + + match consumer { + MqttClientResult::V3((v3_handler, event_loop)) => { + handle_event(&db, self, listening_trigger, v3_handler, event_loop).await + } + MqttClientResult::V5((v5_handler, event_loop)) => { + handle_event(&db, self, listening_trigger, v5_handler, event_loop).await + } + } + } +} + +const TIMEOUT_DURATION: u64 = 10; +const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION); + +fn convert_disconnect_packet_into_string( + disconnect: rumqttc::v5::mqttbytes::v5::Disconnect, +) -> String { + let err_message = disconnect + .properties + .map(|properties| properties.reason_string) + .flatten(); + let reason_code = disconnect.reason_code as u8; + format!( + "Disconnected by the broker, reason code: {}, {}", + reason_code, + err_message + .map(|err| format!("message: {}", err)) + .unwrap_or("".to_string()) + ) +} + +#[async_trait] +pub trait EventLoop { + type Event; + type Error; + + async fn poll(&mut self) -> Result; + async fn verify_connection(&mut self) -> Result<()>; +} + +#[async_trait] +impl EventLoop for V5EventLoop { + type Event = V5Event; + type Error = rumqttc::v5::ConnectionError; + + async fn poll(&mut self) -> Result { + self.poll().await.map_err(|err| to_anyhow(err).into()) + } + + async fn verify_connection(&mut self) -> Result<()> { + let start = std::time::Instant::now(); + + while start.elapsed() < CONNECTION_TIMEOUT { + match self.poll().await.map_err(to_anyhow)? { + Self::Event::Incoming(V5Incoming::ConnAck(_)) => return Ok(()), + Self::Event::Incoming(V5Incoming::Disconnect(disconnect)) => { + return Err(Error::BadConfig(convert_disconnect_packet_into_string( + disconnect, + ))); + } + _ => continue, + } + } + + Err(Error::BadConfig(format!( + "Timeout occurred while trying to connect to mqtt broker after {} seconds", + TIMEOUT_DURATION + ))) + } +} + +#[async_trait] +impl EventLoop for V3EventLoop { + type Event = V3Event; + type Error = rumqttc::ConnectionError; + + async fn poll(&mut self) -> Result { + self.poll().await.map_err(|err| to_anyhow(err).into()) + } + + async fn verify_connection(&mut self) -> Result<()> { + let start = std::time::Instant::now(); + + while start.elapsed() < CONNECTION_TIMEOUT { + match self.poll().await.map_err(to_anyhow)? { + Self::Event::Incoming(rumqttc::Packet::ConnAck(_)) => return Ok(()), + _ => continue, + } + } + + Err(Error::BadConfig(format!( + "Timeout occurred while trying to connect to mqtt broker after {} seconds", + TIMEOUT_DURATION + ))) + } +} + +async fn handle_event( + db: &DB, + listener: &T, + listening_trigger: &ListeningTrigger, + handler: H, + mut event_loop: E, +) -> () +where + T: Listener, + H: MqttEvent, + E: EventLoop, + E::Error: ToString, + ::Payload: From, +{ + loop { + let event = event_loop.poll().await; + + match event { + Ok(event) => { + let publish_data = handler.handle_event(event); + if let Ok(Some((payload, publish_data))) = publish_data { + let trigger_info = HashMap::from([ + ("topic".to_string(), to_raw_value(&publish_data.topic)), + ("retain".to_string(), to_raw_value(&publish_data.retain)), + ("pkid".to_string(), to_raw_value(&publish_data.pkid)), + ("qos".to_string(), to_raw_value(&publish_data.qos)), + ( + "v5".to_string(), + to_raw_value(&publish_data.v5.map(|properties| { + serde_json::json!({ + "payload_format_indicator": properties.payload_format_indicator, + "topic_alias": properties.topic_alias, + "response_topic": properties.response_topic, + "correlation_data": properties.correlation_data.as_deref(), + "user_properties": properties.user_properties, + "subscription_identifiers": properties.subscription_identifiers, + "content_type": properties.content_type, + }) + })), + ), + ]); + let _ = listener + .handle_event(db, listening_trigger, payload.into(), trigger_info, None) + .await; + } + } + Err(err) => { + let error = err.to_string(); + tracing::debug!("Error: {}", &err); + listener + .disable_with_error(db, listening_trigger, error) + .await; + return; + } + } + } +} + +#[derive(Clone)] +#[allow(unused)] +pub struct PublishData { + topic: String, + retain: bool, + pkid: u16, + v5: Option, + qos: u8, +} + +impl PublishData { + pub fn new( + topic: String, + retain: bool, + pkid: u16, + v5: Option, + qos: u8, + ) -> PublishData { + PublishData { topic, retain, pkid, v5, qos } + } +} + +trait MqttEvent { + type IncomingPacket; + type PublishPacket; + type Event; + + fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData; + fn handle_event(&self, event: Self::Event) -> Result>; +} + +impl MqttEvent for V5MqttHandler { + type IncomingPacket = V5Incoming; + type PublishPacket = rumqttc::v5::mqttbytes::v5::Publish; + type Event = V5Event; + + fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData { + PublishData::new( + String::from_utf8(publish_packet.topic.as_ref().to_vec()).unwrap_or("".to_string()), + publish_packet.retain, + publish_packet.pkid, + publish_packet.properties, + publish_packet.qos as u8, + ) + } + + fn handle_event(&self, event: Self::Event) -> Result> { + tracing::debug!("Inside V5 event"); + match event { + Self::Event::Incoming(packet) => match packet { + Self::IncomingPacket::Publish(publish_packet) => { + return Ok(Some(( + publish_packet.payload.clone(), + Self::handle_publish_packet(publish_packet), + ))) + } + Self::IncomingPacket::Disconnect(disconnect) => { + return Err( + anyhow::anyhow!(convert_disconnect_packet_into_string(disconnect)).into(), + ); + } + packet => { + tracing::debug!("Received = {:#?}", packet); + } + }, + Self::Event::Outgoing(packet) => { + tracing::debug!("Outgoing Received = {:#?}", packet); + } + } + + Ok(None) + } +} + +impl MqttEvent for V3MqttHandler { + type IncomingPacket = V3Incoming; + type PublishPacket = rumqttc::mqttbytes::v4::Publish; + type Event = V3Event; + + fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData { + PublishData::new( + publish_packet.topic, + publish_packet.retain, + publish_packet.pkid, + None, + publish_packet.qos as u8, + ) + } + + fn handle_event(&self, event: Self::Event) -> Result> { + tracing::debug!("Inside V3 event"); + match event { + Self::Event::Incoming(packet) => match packet { + Self::IncomingPacket::Publish(publish_packet) => { + return Ok(Some(( + publish_packet.payload.clone(), + Self::handle_publish_packet(publish_packet), + ))) + } + packet => { + tracing::debug!("Received = {:?}", packet); + } + }, + Self::Event::Outgoing(packet) => { + tracing::debug!("Outgoing Received = {:?}", packet); + } + } + + Ok(None) + } +} diff --git a/backend/windmill-api/src/triggers/mqtt/mod.rs b/backend/windmill-api/src/triggers/mqtt/mod.rs index 1ac287b82a..cfc8e36900 100644 --- a/backend/windmill-api/src/triggers/mqtt/mod.rs +++ b/backend/windmill-api/src/triggers/mqtt/mod.rs @@ -11,7 +11,6 @@ use rumqttc::{ AsyncClient as V3AsyncClient, EventLoop as V3EventLoop, MqttOptions as V3MqttOptions, QoS as V3QoS, SubscribeFilter, TlsConfiguration, Transport, }; -use crate::mqtt_triggers::EventLoop; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use sqlx::{types::Json as SqlxJson, FromRow, Type}; @@ -22,9 +21,10 @@ use windmill_common::{ worker::to_raw_value, }; -use crate::triggers::{ trigger_helpers::TriggerJobArgs}; +use crate::triggers::{mqtt::listener::EventLoop, trigger_helpers::TriggerJobArgs}; pub mod handler; +pub mod listener; #[derive(Clone, Copy)] pub struct MqttTrigger; diff --git a/backend/windmill-api/src/triggers/nats/handler_oss.rs b/backend/windmill-api/src/triggers/nats/handler_oss.rs index 6b03dafef0..41f060003f 100644 --- a/backend/windmill-api/src/triggers/nats/handler_oss.rs +++ b/backend/windmill-api/src/triggers/nats/handler_oss.rs @@ -30,7 +30,7 @@ impl TriggerCrud for NatsTrigger { const SUPPORTS_TEST_CONNECTION: bool = false; const ROUTE_PREFIX: &'static str = "/nats_triggers"; const DEPLOYMENT_NAME: &'static str = ""; - const IS_CLOUD_HOSTED: bool = false; + const IS_ALLOWED_ON_CLOUD: bool = false; fn get_deployed_object(path: String) -> DeployedObject { DeployedObject::NatsTrigger { path } diff --git a/backend/windmill-api/src/triggers/nats/listener_oss.rs b/backend/windmill-api/src/triggers/nats/listener_oss.rs new file mode 100644 index 0000000000..80f1befded --- /dev/null +++ b/backend/windmill-api/src/triggers/nats/listener_oss.rs @@ -0,0 +1,41 @@ +#[allow(unused)] + +#[cfg(feature = "private")] +pub use super::listener_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::NatsTrigger, + crate::triggers::{listener::ListeningTrigger, Listener}, + std::sync::Arc, + tokio::sync::RwLock, + windmill_common::{error::Result, jobs::JobTriggerKind, DB}, +}; + +#[cfg(not(feature = "private"))] +#[async_trait::async_trait] +impl Listener for NatsTrigger { + type Consumer = (); + type Extra = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Nats; + + async fn get_consumer( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + Ok(None) + } + async fn consume( + &self, + _db: &DB, + _consumer: Self::Consumer, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) { + () + } +} diff --git a/backend/windmill-api/src/triggers/nats/mod.rs b/backend/windmill-api/src/triggers/nats/mod.rs index b69f2eb3c8..668df073c9 100644 --- a/backend/windmill-api/src/triggers/nats/mod.rs +++ b/backend/windmill-api/src/triggers/nats/mod.rs @@ -2,6 +2,10 @@ mod handler_ee; pub mod handler_oss; +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + #[cfg(feature = "private")] mod mod_ee; #[cfg(feature = "private")] diff --git a/backend/windmill-api/src/postgres_triggers/bool.rs b/backend/windmill-api/src/triggers/postgres/bool.rs similarity index 94% rename from backend/windmill-api/src/postgres_triggers/bool.rs rename to backend/windmill-api/src/triggers/postgres/bool.rs index 9c415780a5..13b0a8fa68 100644 --- a/backend/windmill-api/src/postgres_triggers/bool.rs +++ b/backend/windmill-api/src/triggers/postgres/bool.rs @@ -3,10 +3,10 @@ use thiserror::Error; /** * This implementation is inspired by Postgres replication functionality * from https://github.com/supabase/pg_replicate -* -* Original implementation: +* +* Original implementation: * - https://github.dev/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/bool.rs -* +* */ #[derive(Debug, Error)] diff --git a/backend/windmill-api/src/postgres_triggers/converter.rs b/backend/windmill-api/src/triggers/postgres/converter.rs similarity index 100% rename from backend/windmill-api/src/postgres_triggers/converter.rs rename to backend/windmill-api/src/triggers/postgres/converter.rs diff --git a/backend/windmill-api/src/postgres_triggers/hex.rs b/backend/windmill-api/src/triggers/postgres/hex.rs similarity index 100% rename from backend/windmill-api/src/postgres_triggers/hex.rs rename to backend/windmill-api/src/triggers/postgres/hex.rs diff --git a/backend/windmill-api/src/triggers/postgres/listener.rs b/backend/windmill-api/src/triggers/postgres/listener.rs new file mode 100644 index 0000000000..bfb527e3e7 --- /dev/null +++ b/backend/windmill-api/src/triggers/postgres/listener.rs @@ -0,0 +1,441 @@ +use std::{collections::HashMap, pin::Pin, sync::Arc}; + +use bytes::{BufMut, Bytes, BytesMut}; +use chrono::TimeZone; +use futures::{pin_mut, SinkExt}; +use pg_escape::{quote_identifier, quote_literal}; +use rust_postgres::{Client, CopyBothDuplex, SimpleQueryMessage}; +use tokio::sync::RwLock; +use tokio_stream::StreamExt; +use windmill_common::{ + db::UserDB, + error::{to_anyhow, Error, Result}, + jobs::JobTriggerKind, + worker::to_raw_value, + DB, +}; + +use crate::{ + resources::try_get_resource_from_db_as, + triggers::{ + listener::ListeningTrigger, + postgres::{ + drop_publication, get_default_pg_connection, get_raw_postgres_connection, + handler::drop_logical_replication_slot, + relation::RelationConverter, + replication_message::{ + LogicalReplicationMessage::{ + Begin, Commit, Delete, Insert, Relation, Type, Update, + }, + PrimaryKeepAliveBody, ReplicationMessage, + }, + Postgres, PostgresConfig, PostgresTrigger, ERROR_PUBLICATION_NAME_NOT_EXISTS, + }, + trigger_helpers::TriggerJobArgs, + Listener, + }, +}; + +const ERROR_REPLICATION_SLOT_NOT_EXISTS: &str = r#"The replication slot associated with this trigger no longer exists. Recreate a new replication slot or select an existing one in the advanced tab, or delete and recreate a new trigger"#; + +pub struct LogicalReplicationSettings { + pub streaming: bool, +} + +impl LogicalReplicationSettings { + pub fn new(streaming: bool) -> Self { + Self { streaming } + } +} + +pub struct PostgresSimpleClient(Client); + +trait RowExist { + fn row_exist(&self) -> bool; +} + +impl RowExist for Vec { + fn row_exist(&self) -> bool { + self.iter() + .find_map(|element| { + if let SimpleQueryMessage::CommandComplete(value) = element { + Some(*value) + } else { + None + } + }) + .is_some_and(|value| value > 0) + } +} + +impl PostgresSimpleClient { + async fn new(database: &Postgres) -> Result { + let client = get_raw_postgres_connection(database, true).await?; + + Ok(PostgresSimpleClient(client)) + } + + async fn execute_query( + &self, + query: &str, + ) -> std::result::Result, rust_postgres::Error> { + self.0.simple_query(query).await + } + + async fn get_logical_replication_stream( + &self, + publication_name: &str, + logical_replication_slot_name: &str, + ) -> Result<(CopyBothDuplex, LogicalReplicationSettings)> { + let options = format!( + r#"("proto_version" '2', "publication_names" {})"#, + quote_literal(publication_name), + ); + + let query = format!( + r#"START_REPLICATION SLOT {} LOGICAL 0/0 {}"#, + quote_identifier(logical_replication_slot_name), + options + ); + + Ok(( + self.0 + .copy_both_simple::(query.as_str()) + .await + .map_err(to_anyhow)?, + LogicalReplicationSettings::new(false), + )) + } + + async fn send_status_update( + primary_keep_alive: PrimaryKeepAliveBody, + copy_both_stream: &mut Pin<&mut CopyBothDuplex>, + ) { + let mut buf = BytesMut::new(); + let ts = chrono::Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let ts = chrono::Utc::now() + .signed_duration_since(ts) + .num_microseconds() + .unwrap_or(0); + + buf.put_u8(b'r'); + buf.put_u64(primary_keep_alive.wal_end); + buf.put_u64(primary_keep_alive.wal_end); + buf.put_u64(primary_keep_alive.wal_end); + buf.put_i64(ts); + buf.put_u8(0); + copy_both_stream.send(buf.freeze()).await.unwrap(); + tracing::debug!("Send update status message"); + } +} + +#[async_trait::async_trait] +impl Listener for PostgresTrigger { + type Consumer = (CopyBothDuplex, LogicalReplicationSettings); + type Extra = (); + type ExtraState = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Postgres; + + async fn get_consumer( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + let ListeningTrigger:: { workspace_id, trigger_config, .. } = + listening_trigger; + + let PostgresConfig { + postgres_resource_path, publication_name, replication_slot_name, .. + } = trigger_config; + + let authed = listening_trigger + .authed(db, &Self::TRIGGER_KIND.to_string()) + .await?; + + let database = try_get_resource_from_db_as::( + &authed, + Some(UserDB::new(db.clone())), + &db, + postgres_resource_path, + workspace_id, + ) + .await?; + + let client = PostgresSimpleClient::new(&database).await?; + + let publication = client + .execute_query(&format!( + "SELECT pubname FROM pg_publication WHERE pubname = {}", + quote_literal(&publication_name) + )) + .await + .map_err(to_anyhow)?; + + if !publication.row_exist() { + return Err(Error::BadConfig( + ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(), + )); + } + + let replication_slot = client + .execute_query(&format!( + "SELECT slot_name FROM pg_replication_slots WHERE slot_name = {}", + quote_literal(&replication_slot_name) + )) + .await + .map_err(to_anyhow)?; + + if !replication_slot.row_exist() { + return Err(Error::BadConfig( + ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string(), + )); + } + + let (logical_replication_stream, logical_replication_settings) = client + .get_logical_replication_stream(&publication_name, &replication_slot_name) + .await + .map_err(to_anyhow)?; + + Ok(Some(( + logical_replication_stream, + logical_replication_settings, + ))) + } + async fn consume( + &self, + db: &DB, + consumer: Self::Consumer, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, + ) { + let (logical_replication_stream, logical_replication_settings) = consumer; + pin_mut!(logical_replication_stream); + let mut relations = RelationConverter::new(); + tracing::info!( + "Starting to listen for postgres trigger {}", + &listening_trigger.path + ); + loop { + let message = logical_replication_stream.next().await; + let message = match message { + Some(message) => message, + None => { + tracing::error!( + "Stream for postgres trigger {} closed", + &listening_trigger.path + ); + if let None = self + .update_ping_and_loop_ping_status( + db, + listening_trigger, + err_message.clone(), + Some("Stream closed".to_string()), + ) + .await + { + return; + } + return; + } + }; + + let message = match message { + Ok(message) => message, + Err(err) => { + let err = format!( + "Postgres trigger named {} had an error while receiving a message : {}", + &listening_trigger.path, + err.to_string() + ); + self.disable_with_error(db, listening_trigger, err).await; + return; + } + }; + + let logical_message = match ReplicationMessage::parse(message) { + Ok(logical_message) => logical_message, + Err(err) => { + let err = format!( + "Postgres trigger named: {} had an error while parsing message: {}", + &listening_trigger.path, + err.to_string() + ); + self.disable_with_error(db, listening_trigger, err).await; + return; + } + }; + + match logical_message { + ReplicationMessage::PrimaryKeepAlive(primary_keep_alive) => { + if primary_keep_alive.reply { + PostgresSimpleClient::send_status_update( + primary_keep_alive, + &mut logical_replication_stream, + ) + .await; + } + } + ReplicationMessage::XLogData(x_log_data) => { + let logical_replication_message = match x_log_data + .parse(&logical_replication_settings) + { + Ok(logical_replication_message) => logical_replication_message, + Err(err) => { + tracing::error!("Postgres trigger named: {} had an error while trying to parse incomming stream message: {}", &listening_trigger.path, err.to_string()); + continue; + } + }; + + let json = match logical_replication_message { + Relation(relation_body) => { + relations.add_relation(relation_body); + None + } + Begin | Type | Commit => None, + Insert(insert) => Some(( + insert.o_id, + Ok(None), + relations.row_to_json((insert.o_id, insert.tuple)), + "insert", + )), + Update(update) => { + let old_row = update + .old_tuple + .map(|old_tuple| relations.row_to_json((update.o_id, old_tuple))) + .transpose(); + let row = relations.row_to_json((update.o_id, update.new_tuple)); + Some((update.o_id, old_row, row, "update")) + } + Delete(delete) => { + let row = delete + .old_tuple + .unwrap_or_else(|| delete.key_tuple.unwrap()); + Some(( + delete.o_id, + Ok(None), + relations.row_to_json((delete.o_id, row)), + "delete", + )) + } + }; + match json { + Some((o_id, Ok(old_row), Ok(row), transaction_type)) => { + let relation = match relations.get_relation(o_id) { + Ok(relation) => relation, + Err(err) => { + tracing::error!( + "Postgres trigger named: {}, error: {}", + &listening_trigger.path, + err.to_string() + ); + continue; + } + }; + let database_info = HashMap::from([ + ("schema_name".to_string(), to_raw_value(&relation.namespace)), + ("table_name".to_string(), to_raw_value(&relation.name)), + ( + "transaction_type".to_string(), + to_raw_value(&transaction_type), + ), + ("old_row".to_string(), to_raw_value(&old_row)), + ("row".to_string(), to_raw_value(&row)), + ]); + let _ = self + .handle_event( + db, + listening_trigger, + database_info, + HashMap::new(), + None, + ) + .await; + } + Some((o_id, old_row, row, transaction_type)) => { + let relation = match relations.get_relation(o_id) { + Ok(relation) => relation, + Err(err) => { + tracing::error!( + "Postgres trigger named: {}, error: {}", + &listening_trigger.path, + err.to_string() + ); + continue; + } + }; + + if let Err(err) = old_row { + tracing::error!( + transaction_type = ?transaction_type, + schema = %relation.namespace, + table = %relation.name, + error = %err, + "Failed to decode OLD row for {} transaction on {}.{}", + transaction_type, + relation.namespace, + relation.name, + ); + } + + if let Err(err) = row { + tracing::error!( + transaction_type = ?transaction_type, + schema = %relation.namespace, + table = %relation.name, + error = %err, + "Failed to decode NEW row for {} transaction on {}.{}", + transaction_type, + relation.namespace, + relation.name, + ); + } + } + _ => {} + } + } + } + } + } + + async fn cleanup( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + _extra_state: Option<&Self::ExtraState>, + ) -> Result<()> { + let authed = listening_trigger + .authed(db, &Self::TRIGGER_KIND.to_string()) + .await?; + + let user_db = UserDB::new(db.clone()); + + let mut pg_connection = get_default_pg_connection( + authed, + Some(user_db), + &db, + &listening_trigger.trigger_config.postgres_resource_path, + &listening_trigger.workspace_id, + ) + .await?; + + if listening_trigger.trigger_config.basic_mode.unwrap_or(false) { + drop_logical_replication_slot( + &mut pg_connection, + &listening_trigger.trigger_config.replication_slot_name, + ) + .await?; + + drop_publication( + &mut pg_connection, + &listening_trigger.trigger_config.publication_name, + ) + .await?; + } + + Ok(()) + } +} diff --git a/backend/windmill-api/src/triggers/postgres/mod.rs b/backend/windmill-api/src/triggers/postgres/mod.rs index d94b08f67d..191c57b166 100644 --- a/backend/windmill-api/src/triggers/postgres/mod.rs +++ b/backend/windmill-api/src/triggers/postgres/mod.rs @@ -22,8 +22,14 @@ use windmill_common::{ utils::empty_as_none, }; +mod bool; +mod converter; pub mod handler; +mod hex; +pub mod listener; mod mapper; +mod relation; +mod replication_message; #[derive(Clone, Copy)] pub struct PostgresTrigger; diff --git a/backend/windmill-api/src/postgres_triggers/relation.rs b/backend/windmill-api/src/triggers/postgres/relation.rs similarity index 100% rename from backend/windmill-api/src/postgres_triggers/relation.rs rename to backend/windmill-api/src/triggers/postgres/relation.rs diff --git a/backend/windmill-api/src/postgres_triggers/replication_message.rs b/backend/windmill-api/src/triggers/postgres/replication_message.rs similarity index 99% rename from backend/windmill-api/src/postgres_triggers/replication_message.rs rename to backend/windmill-api/src/triggers/postgres/replication_message.rs index e33f4bbf86..5ae7f8cf2f 100644 --- a/backend/windmill-api/src/postgres_triggers/replication_message.rs +++ b/backend/windmill-api/src/triggers/postgres/replication_message.rs @@ -12,7 +12,7 @@ use bytes::Bytes; use rust_postgres::types::{Oid, Type}; use thiserror::Error; -use super::trigger::LogicalReplicationSettings; +use super::listener::LogicalReplicationSettings; const PRIMARY_KEEPALIVE_BYTE: u8 = b'k'; const X_LOG_DATA_BYTE: u8 = b'w'; diff --git a/backend/windmill-api/src/triggers/sqs/handler_oss.rs b/backend/windmill-api/src/triggers/sqs/handler_oss.rs index 53365def58..6b6d18b72e 100644 --- a/backend/windmill-api/src/triggers/sqs/handler_oss.rs +++ b/backend/windmill-api/src/triggers/sqs/handler_oss.rs @@ -31,7 +31,7 @@ impl TriggerCrud for SqsTrigger { const SUPPORTS_TEST_CONNECTION: bool = false; const ROUTE_PREFIX: &'static str = "/sqs_triggers"; const DEPLOYMENT_NAME: &'static str = ""; - const IS_CLOUD_HOSTED: bool = false; + const IS_ALLOWED_ON_CLOUD: bool = false; fn get_deployed_object(path: String) -> DeployedObject { DeployedObject::SqsTrigger { path } diff --git a/backend/windmill-api/src/triggers/sqs/listener_oss.rs b/backend/windmill-api/src/triggers/sqs/listener_oss.rs new file mode 100644 index 0000000000..0aaf5360c3 --- /dev/null +++ b/backend/windmill-api/src/triggers/sqs/listener_oss.rs @@ -0,0 +1,41 @@ +#[allow(unused)] + +#[cfg(feature = "private")] +pub use super::listener_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::SqsTrigger, + crate::triggers::{listener::ListeningTrigger, Listener}, + std::sync::Arc, + tokio::sync::RwLock, + windmill_common::{error::Result, jobs::JobTriggerKind, DB}, +}; + +#[cfg(not(feature = "private"))] +#[async_trait::async_trait] +impl Listener for SqsTrigger { + type Consumer = (); + type Extra = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Sqs; + + async fn get_consumer( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + Ok(None) + } + async fn consume( + &self, + _db: &DB, + _consumer: Self::Consumer, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) { + () + } +} diff --git a/backend/windmill-api/src/triggers/sqs/mod.rs b/backend/windmill-api/src/triggers/sqs/mod.rs index 74a93c12a5..8c7217725c 100644 --- a/backend/windmill-api/src/triggers/sqs/mod.rs +++ b/backend/windmill-api/src/triggers/sqs/mod.rs @@ -2,6 +2,10 @@ mod handler_ee; pub mod handler_oss; +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + #[cfg(feature = "private")] mod mod_ee; #[cfg(feature = "private")] diff --git a/backend/windmill-api/src/triggers/trigger_helpers.rs b/backend/windmill-api/src/triggers/trigger_helpers.rs index 8901c24fcc..7deaf6dc1d 100644 --- a/backend/windmill-api/src/triggers/trigger_helpers.rs +++ b/backend/windmill-api/src/triggers/trigger_helpers.rs @@ -46,7 +46,7 @@ struct ScriptInfo { #[derive(Debug, Deserialize)] struct PropertyDefinition { - r#type: Option, + r#type: Option>, } #[derive(Debug, Deserialize)] @@ -119,7 +119,11 @@ fn runnable_format_from_schema_without_preprocessor( if schema.as_ref().is_some_and(|schema| { schema.properties.as_ref().is_some_and(|properties| { properties.iter().any(|(key, def)| { - key == "payload" && def.r#type.as_ref().is_some_and(|t| t == "array") + key == "payload" + && def.r#type.as_ref().is_some_and(|t| { + let typ = t.get().trim(); + typ == "array" || (typ.starts_with('[') && typ.ends_with(']')) + }) }) }) }) => @@ -450,6 +454,7 @@ pub trait TriggerJobArgs { trigger_info: HashMap>, ) -> impl Future> + Send { async move { + tracing::debug!("Building job args for {runnable_id:?}"); let runnable_format = get_runnable_format(runnable_id, w_id, db, &Self::TRIGGER_KIND).await?; let job_args = match runnable_format { @@ -460,6 +465,7 @@ pub trait TriggerJobArgs { Self::build_job_args_v2(has_preprocessor, &payload, trigger_info) } }; + Ok(job_args) } } @@ -475,7 +481,7 @@ pub trait TriggerJobArgs { } #[allow(dead_code)] -async fn trigger_runnable_inner( +pub async fn trigger_runnable_inner( db: &DB, user_db: Option, authed: ApiAuthed, @@ -487,6 +493,7 @@ async fn trigger_runnable_inner( error_handler_path: Option<&str>, error_handler_args: Option<&sqlx::types::Json>>, trigger_path: String, + job_id: Option, ) -> Result<(Uuid, Option, Option)> { let error_handler_args = error_handler_args.map(|args| { let args = args @@ -499,7 +506,7 @@ async fn trigger_runnable_inner( let user_db = user_db.unwrap_or_else(|| UserDB::new(db.clone())); let (uuid, delete_after_use, early_return) = if is_flow { - let run_query = RunJobQuery::default(); + let run_query = RunJobQuery { job_id, ..Default::default() }; let path = StripPath(runnable_path.to_string()); let (uuid, early_return) = run_flow_by_path_inner( authed, @@ -524,6 +531,7 @@ async fn trigger_runnable_inner( error_handler_path, error_handler_args.as_ref(), trigger_path, + job_id, ) .await?; (uuid, delete_after_use, None) @@ -545,6 +553,7 @@ pub async fn trigger_runnable( error_handler_path: Option<&str>, error_handler_args: Option<&sqlx::types::Json>>, trigger_path: String, + job_id: Option, ) -> Result { let (uuid, _, _) = trigger_runnable_inner( db, @@ -558,6 +567,7 @@ pub async fn trigger_runnable( error_handler_path, error_handler_args, trigger_path, + job_id, ) .await?; Ok((StatusCode::CREATED, uuid.to_string()).into_response()) @@ -590,6 +600,7 @@ pub async fn trigger_runnable_and_wait_for_result( error_handler_path, error_handler_args, trigger_path, + None, ) .await?; let (result, success) = @@ -616,7 +627,7 @@ pub async fn trigger_runnable_and_wait_for_raw_result( error_handler_path: Option<&str>, error_handler_args: Option<&sqlx::types::Json>>, trigger_path: String, -) -> Result> { +) -> Result<(Box, bool)> { let username = authed.username.clone(); let (uuid, delete_after_use, early_return) = trigger_runnable_inner( db, @@ -630,6 +641,7 @@ pub async fn trigger_runnable_and_wait_for_raw_result( error_handler_path, error_handler_args, trigger_path, + None, ) .await?; @@ -648,6 +660,37 @@ pub async fn trigger_runnable_and_wait_for_raw_result( delete_job_metadata_after_use(&db, uuid).await?; } + Ok((result, success)) +} + +pub async fn trigger_runnable_and_wait_for_raw_result_with_error_ctx( + db: &DB, + user_db: Option, + authed: ApiAuthed, + workspace_id: &str, + runnable_path: &str, + is_flow: bool, + args: PushArgsOwned, + retry: Option<&sqlx::types::Json>, + error_handler_path: Option<&str>, + error_handler_args: Option<&sqlx::types::Json>>, + trigger_path: String, +) -> Result> { + let (result, success) = trigger_runnable_and_wait_for_raw_result( + db, + user_db, + authed, + workspace_id, + runnable_path, + is_flow, + args, + retry, + error_handler_path, + error_handler_args, + trigger_path, + ) + .await?; + if !success { Err(windmill_common::error::Error::internal_err(format!( "{} {runnable_path} failed: {:?}", @@ -670,9 +713,10 @@ async fn trigger_script_internal( error_handler_path: Option<&str>, error_handler_args: Option<&sqlx::types::Json>>>, trigger_path: String, + job_id: Option, ) -> Result<(Uuid, Option)> { if retry.is_none() && error_handler_path.is_none() { - let run_query = RunJobQuery::default(); + let run_query = RunJobQuery { job_id, ..Default::default() }; let path = StripPath(script_path.to_string()); run_script_by_path_inner( authed, @@ -696,6 +740,7 @@ async fn trigger_script_internal( error_handler_path, error_handler_args, trigger_path, + job_id, ) .await } @@ -712,6 +757,7 @@ async fn trigger_script_with_retry_and_error_handler( error_handler_path: Option<&str>, error_handler_args: Option<&sqlx::types::Json>>>, trigger_path: String, + job_id: Option, ) -> Result<(Uuid, Option)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -762,17 +808,21 @@ async fn trigger_script_with_retry_and_error_handler( custom_concurrency_key, concurrent_limit, concurrency_time_window_s, + custom_debounce_key, + debounce_delay_s, cache_ttl, priority, apply_preprocessor, .. - } => JobPayload::SingleScriptFlow { + } => JobPayload::SingleStepFlow { path, - hash, + hash: Some(hash), + flow_version: None, args: HashMap::from(&push_args), retry, error_handler_path, error_handler_args, + skip_handler: None, custom_concurrency_key, concurrent_limit, concurrency_time_window_s, @@ -781,6 +831,8 @@ async fn trigger_script_with_retry_and_error_handler( tag_override: tag.clone(), apply_preprocessor, trigger_path: Some(trigger_path), + custom_debounce_key, + debounce_delay_s, }, _ => { return Err(windmill_common::error::Error::internal_err(format!( @@ -805,7 +857,7 @@ async fn trigger_script_with_retry_and_error_handler( None, None, None, - None, + job_id, false, false, None, @@ -816,6 +868,8 @@ async fn trigger_script_with_retry_and_error_handler( None, push_authed.as_ref(), false, + None, + None, ) .await?; tx.commit().await?; diff --git a/backend/windmill-api/src/triggers/websocket/handler.rs b/backend/windmill-api/src/triggers/websocket/handler.rs index 0a321a450c..510f6e97ae 100644 --- a/backend/windmill-api/src/triggers/websocket/handler.rs +++ b/backend/windmill-api/src/triggers/websocket/handler.rs @@ -12,6 +12,7 @@ use tokio_tungstenite::connect_async; use windmill_common::{ db::UserDB, error::{Error, Result}, + worker::to_raw_value, }; use windmill_git_sync::DeployedObject; @@ -40,6 +41,7 @@ impl TriggerCrud for WebsocketTrigger { "initial_messages", "url_runnable_args", "can_return_message", + "can_return_error_result", ]; const IS_ALLOWED_ON_CLOUD: bool = false; @@ -105,13 +107,14 @@ impl TriggerCrud for WebsocketTrigger { url_runnable_args, edited_by, can_return_message, + can_return_error_result, email, edited_at, error_handler_path, error_handler_args, retry ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13, $14, $15 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), $14, $15, $16 ) "#, w_id, @@ -128,6 +131,7 @@ impl TriggerCrud for WebsocketTrigger { .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) as _, authed.username, trigger.config.can_return_message, + trigger.config.can_return_error_result, authed.email, trigger.error_handling.error_handler_path, trigger.error_handling.error_handler_args as _, @@ -177,14 +181,15 @@ impl TriggerCrud for WebsocketTrigger { edited_by = $8, email = $9, can_return_message = $10, + can_return_error_result = $11, edited_at = now(), server_id = NULL, error = NULL, - error_handler_path = $13, - error_handler_args = $14, - retry = $15 + error_handler_path = $14, + error_handler_args = $15, + retry = $16 WHERE - workspace_id = $11 AND path = $12 + workspace_id = $12 AND path = $13 ", trigger.config.url, trigger.base.script_path, @@ -200,6 +205,7 @@ impl TriggerCrud for WebsocketTrigger { &authed.username, &authed.email, trigger.config.can_return_message, + trigger.config.can_return_error_result, w_id, path, trigger.error_handling.error_handler_path, @@ -231,7 +237,7 @@ impl TriggerCrud for WebsocketTrigger { url.starts_with("$flow:"), &db, authed.clone(), - config.url_runnable_args.as_ref(), + config.url_runnable_args.as_ref().map(to_raw_value).as_ref(), &workspace_id, ) .await?, diff --git a/backend/windmill-api/src/triggers/websocket/listener.rs b/backend/windmill-api/src/triggers/websocket/listener.rs new file mode 100644 index 0000000000..bfecaf479d --- /dev/null +++ b/backend/windmill-api/src/triggers/websocket/listener.rs @@ -0,0 +1,549 @@ +use super::WebsocketTrigger; +use crate::triggers::{ + listener::ListeningTrigger, + trigger_helpers::{ + trigger_runnable, trigger_runnable_and_wait_for_raw_result, + trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, + }, + websocket::{get_url_from_runnable_value, WebsocketConfig}, + Listener, +}; +use anyhow::Context; +use async_trait::async_trait; +use futures::{stream::SplitSink, SinkExt, StreamExt}; +use http::Response; +use itertools::Itertools; +use serde::{ + de::{self, MapAccess, Visitor}, + Deserialize, Deserializer, +}; +use serde_json::{value::RawValue, Value}; +use std::{borrow::Cow, collections::HashMap, fmt, sync::Arc}; +use tokio::{net::TcpStream, sync::RwLock}; +use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; +use windmill_common::{ + error::{to_anyhow, Error, Result}, + jobs::JobTriggerKind, + utils::report_critical_error, + worker::to_raw_value, + DB, +}; +use windmill_queue::PushArgsOwned; + +impl ListeningTrigger { + async fn send_initial_messages( + &self, + writer: &mut SplitSink>, Message>, + db: &DB, + ) -> Result<()> { + let initial_messages: Vec = self + .trigger_config + .initial_messages + .as_deref() + .unwrap_or_default() + .iter() + .filter_map(|m| serde_json::from_str(m.get()).ok()) + .collect_vec(); + + let WebsocketConfig { ref url, .. } = self.trigger_config; + let runnable_kind = if self.is_flow { "flow" } else { "script" }; + let mut authed_o = None; + for start_message in initial_messages { + match start_message { + InitialMessage::RawMessage(msg) => { + let msg = if msg.starts_with("\"") && msg.ends_with("\"") { + msg[1..msg.len() - 1].to_string() + } else { + msg + }; + tracing::info!( + "Sending raw message initial message to WebSocket {}: {}", + url, + msg + ); + writer + .send(tokio_tungstenite::tungstenite::Message::Text(msg)) + .await + .map_err(to_anyhow) + .with_context(|| "failed to send raw message")?; + } + InitialMessage::RunnableResult { path, is_flow, args } => { + tracing::info!( + "Running {} {} for initial message to WebSocket {}", + runnable_kind, + path, + url, + ); + + let args = raw_value_to_args_hashmap(Some(&args))?; + + if authed_o.is_none() { + authed_o = Some(self.authed(db, "ws").await?); + } + let authed = authed_o.clone().unwrap(); + + let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx( + db, + None, + authed.clone(), + &self.workspace_id, + &path, + is_flow, + PushArgsOwned { args, extra: None }, + None, + None, + None, + "".to_string(), // doesn't matter as no retry/error handler + ) + .await + .map(|r| r.get().to_owned())?; + + tracing::info!( + "Sending {} {} result to WebSocket {}", + runnable_kind, + path, + url + ); + + // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. + // it falls back to the original serialized JSON if it doesn't work. + let result = serde_json::from_str::(result.as_str()).unwrap_or(result); + + writer + .send(tokio_tungstenite::tungstenite::Message::Text(result)) + .await + .map_err(to_anyhow) + .with_context(|| { + format!("Failed to send {} {} result", runnable_kind, path) + })?; + } + } + } + Ok(()) + } +} + +#[async_trait] +impl Listener for WebsocketTrigger { + type Consumer = ( + WebSocketStream>, + Response>>, + ); + type Extra = ReturnMessageChannels; + type ExtraState = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Websocket; + async fn get_consumer( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + let url = &listening_trigger.trigger_config.url; + let connect_url: Cow = if url.starts_with("$") { + if url.starts_with("$flow:") || url.starts_with("$script:") { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return Ok(None); + }, + _ = self.loop_ping(&db, listening_trigger, err_message.clone(), Some( + "Waiting on runnable to return WebSocket URL...".to_string() + )) => { + return Ok(None); + }, + url_result = { + let authed = listening_trigger.authed(db, "ws").await?; + let args = listening_trigger.trigger_config.url_runnable_args.as_ref().map(|r| &r.0); + let path = url.splitn(2, ':').nth(1).unwrap(); + get_url_from_runnable_value(path, url.starts_with("$flow:"), db, authed, args, &listening_trigger.workspace_id) + } => match url_result { + Ok(url) => Cow::Owned(url), + Err(err) => { + return Err(anyhow::anyhow!("Error getting WebSocket URL from runnable after 5 tries: {:?}", err).into()); + } + }, + } + } else { + return Err(anyhow::anyhow!("Invalid WebSocket runnable path: {}", url).into()); + } + } else { + Cow::Borrowed(&url) + }; + + let connection = connect_async(connect_url.as_ref()) + .await + .map(|conn| Some(conn)) + .map_err(|err| to_anyhow(err).into()); + + connection + } + async fn consume( + &self, + db: &DB, + consumer: Self::Consumer, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, + ) { + let WebsocketConfig { ref url, .. } = listening_trigger.trigger_config; + + tracing::info!("Connected to WebSocket {}", url); + + let (ws_stream, _) = consumer; + + let (mut writer, mut reader) = ws_stream.split(); + + // send initial messages + if listening_trigger.trigger_mode { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return; + }, + _ = self.loop_ping(db, listening_trigger, err_message.clone(), Some("Sending initial messages...".to_string())) => { + return; + }, + result = listening_trigger.send_initial_messages(&mut writer, &db) => { + if let Err(err) = result { + self.disable_with_error(&db, listening_trigger, format!("Error sending initial messages: {:?}", err)).await; + return + } else { + tracing::debug!("Initial messages sent successfully to WebSocket {}", url); + } + } + } + } + + let (return_message_channels, message_sender_handle) = if listening_trigger.trigger_mode + && listening_trigger.trigger_config.can_return_message + { + let (send_message_tx, mut rx) = tokio::sync::mpsc::channel::(100); + let w_id = listening_trigger.workspace_id.clone(); + let url = url.clone(); + let db = db.clone(); + let handle = tokio::spawn(async move { + while let Some(message) = rx.recv().await { + if let Err(err) = writer + .send(tokio_tungstenite::tungstenite::Message::Text(message)) + .await + { + report_critical_error(format!("Could not send runnable result to WebSocket {} because of error: {}", url, err), db.clone(), Some(&w_id), None).await; + } + } + }); + + let killpill_rx = killpill_rx.resubscribe(); + + let return_message_channels = ReturnMessageChannels { send_message_tx, killpill_rx }; + + (Some(return_message_channels), Some(handle)) + } else { + (None, None) + }; + + tokio::select! { + biased; + _ = killpill_rx.recv() => { + }, + _ = self.loop_ping(db, listening_trigger, err_message.clone(), None) => { + }, + _ = async { + let filters: Vec = if listening_trigger.trigger_mode { + listening_trigger + .trigger_config + .filters + .iter() + .filter_map(|m| serde_json::from_str(m.get()).ok()) + .collect_vec() + } else { + vec![] + }; + loop { + if let Some(msg) = reader.next().await { + match msg { + Ok(msg) => { + match msg { + tokio_tungstenite::tungstenite::Message::Text(text) => { + tracing::debug!("Received text message from WebSocket {}: {}", url, text); + let mut should_handle = true; + for filter in &filters { + match filter { + Filter::JsonFilter(JsonFilter { key, value }) => { + let mut deserializer = serde_json::Deserializer::from_str(text.as_str()); + should_handle = match is_value_superset(&mut deserializer, key, &value) { + Ok(filter_match) => { + filter_match + }, + Err(err) => { + tracing::warn!("Error deserializing filter for WebSocket {}: {:?}", url, err); + false + } + }; + } + } + if !should_handle { + break; + } + } + if should_handle { + let trigger_info = HashMap::from([ + ("url".to_string(), to_raw_value(&listening_trigger.trigger_config.url)), + ]); + let _ = self.handle_event(db, listening_trigger, text, trigger_info, return_message_channels.clone()).await; + } + }, + a @ _ => { + tracing::debug!("Received non text-message from WebSocket {}: {:?}", url, a); + } + } + }, + Err(err) => { + tracing::error!("Error reading from WebSocket {}: {:?}", url, err); + } + } + } else { + tracing::error!("WebSocket {} closed", url); + self.update_ping_and_loop_ping_status(db, listening_trigger, err_message.clone(), Some("WebSocket closed".to_string())).await; + break; + } + } + } => {} + } + // make sure to stop return message handler + if let Some(message_sender_handle) = message_sender_handle { + message_sender_handle.abort(); + } + } + + async fn handle_trigger( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + payload: Self::Payload, + trigger_info: HashMap>, + extra: Option, + ) -> Result<()> { + let ListeningTrigger { + path, + is_flow, + workspace_id, + trigger_config, + script_path, + error_handling, + .. + } = listening_trigger; + + let WebsocketConfig { url, .. } = trigger_config; + + let args = WebsocketTrigger::build_job_args( + &script_path, + *is_flow, + workspace_id, + db, + payload, + trigger_info, + ) + .await; + + let args = match args { + Ok(args) => args, + Err(err) => { + return Err(err); + } + }; + + let authed = listening_trigger.authed(db, "ws").await?; + + let (retry, error_handler_path, error_handler_args) = match error_handling.as_ref() { + Some(error_handling) => ( + error_handling.retry.as_ref(), + error_handling.error_handler_path.as_deref(), + error_handling.error_handler_args.as_ref(), + ), + None => (None, None, None), + }; + if let Some(ReturnMessageChannels { send_message_tx, mut killpill_rx }) = extra { + let db_ = db.clone(); + let url = url.to_owned(); + let script_path = script_path.to_owned(); + let is_flow = *is_flow; + let w_id = workspace_id.to_owned(); + let retry = retry.cloned(); + let error_handler_path = error_handler_path.map(|s| s.to_string()); + let error_handler_args = error_handler_args.cloned(); + let trigger_path = path.clone(); + let can_return_error_result = trigger_config.can_return_error_result; + let handle_response_f = async move { + tokio::select! { + _ = killpill_rx.recv() => { + return; + }, + result = trigger_runnable_and_wait_for_raw_result( + &db_, + None, + authed, + &w_id, + &script_path, + is_flow, + args, + retry.as_ref(), + error_handler_path.as_deref(), + error_handler_args.as_ref(), + format!("websocket_trigger/{}", trigger_path), + ) => { + if let Ok((result, success)) = result { + if !success && !can_return_error_result { + return; + } + let result = result.get().to_owned(); + // only send the result if it's not null + if result != "null" { + tracing::info!("Sending job result to WebSocket {}", url); + // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. + // it falls back to the original serialized JSON if it doesn't work. + let result = serde_json::from_str::(result.as_str()).unwrap_or(result); + if let Err(err) = send_message_tx.send(result).await { + report_critical_error(format!("Could not send runnable result to WebSocket {} because of error: {}", url, err), db_.clone(), Some(&w_id), None).await; + } + } + } + } + }; + }; + + tokio::spawn(handle_response_f); + } else { + trigger_runnable( + db, + None, + authed, + &workspace_id, + &script_path, + *is_flow, + args, + retry, + error_handler_path, + error_handler_args, + format!("websocket_trigger/{}", listening_trigger.path), + None, + ) + .await?; + } + + Ok(()) + } +} + +#[derive(Deserialize)] +pub struct JsonFilter { + key: String, + value: Value, +} + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum Filter { + JsonFilter(JsonFilter), +} + +pub struct ReturnMessageChannels { + send_message_tx: tokio::sync::mpsc::Sender, + killpill_rx: tokio::sync::broadcast::Receiver<()>, +} + +impl Clone for ReturnMessageChannels { + fn clone(&self) -> Self { + Self { + send_message_tx: self.send_message_tx.clone(), + killpill_rx: self.killpill_rx.resubscribe(), + } + } +} + +#[derive(Debug, Deserialize)] +enum InitialMessage { + #[serde(rename = "raw_message")] + RawMessage(String), + #[serde(rename = "runnable_result")] + RunnableResult { path: String, args: Box, is_flow: bool }, +} + +struct SupersetVisitor<'a> { + key: &'a str, + value_to_check: &'a Value, +} + +impl<'de, 'a> Visitor<'de> for SupersetVisitor<'a> { + type Value = bool; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a JSON object with a specific key at the top level") + } + + fn visit_map(self, mut map: V) -> std::result::Result + where + V: MapAccess<'de>, + { + while let Some(key) = map.next_key::()? { + if key == self.key { + // Deserialize the value for the key and check if it's a superset + let json_value: Value = map.next_value()?; + return Ok(is_superset(&json_value, self.value_to_check)); + } else { + // Skip the value if it's not the one we're interested in + let _ = map.next_value::()?; + } + } + // If the key was not found, return false + Ok(false) + } +} + +fn is_superset(json_value: &Value, value_to_check: &Value) -> bool { + match (json_value, value_to_check) { + (Value::Object(json_map), Value::Object(check_map)) => { + // Check that all keys and values in check_map exist and match in json_map + check_map.iter().all(|(k, v)| { + json_map + .get(k) + .map_or(false, |json_val| is_superset(json_val, v)) + }) + } + (Value::Array(json_array), Value::Array(check_array)) => { + // Check that all elements in check_array exist in json_array + check_array.iter().all(|check_item| { + json_array + .iter() + .any(|json_item| is_superset(json_item, check_item)) + }) + } + _ => json_value == value_to_check, + } +} + +// A function to deserialize and check if the value at the given key is a superset of a passed value +fn is_value_superset<'a, 'de, D>( + deserializer: D, + key: &'a str, + value_to_check: &'a Value, +) -> std::result::Result +where + D: Deserializer<'de>, +{ + deserializer.deserialize_map(SupersetVisitor { key, value_to_check }) +} + +fn raw_value_to_args_hashmap( + args: Option<&Box>, +) -> Result>> { + let args = if let Some(args) = args { + serde_json::from_str::>>>(args.get()) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? + .unwrap_or_else(HashMap::new) + } else { + HashMap::new() + }; + Ok(args) +} diff --git a/backend/windmill-api/src/triggers/websocket/mod.rs b/backend/windmill-api/src/triggers/websocket/mod.rs index c367811406..7a4d98bcc1 100644 --- a/backend/windmill-api/src/triggers/websocket/mod.rs +++ b/backend/windmill-api/src/triggers/websocket/mod.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use crate::{ db::ApiAuthed, - triggers::trigger_helpers::{trigger_runnable_and_wait_for_raw_result, TriggerJobArgs}, + triggers::trigger_helpers::{trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs}, }; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; @@ -16,6 +16,7 @@ use windmill_common::{ use windmill_queue::PushArgsOwned; mod handler; +mod listener; #[derive(Copy, Clone)] pub struct WebsocketTrigger; @@ -39,6 +40,8 @@ pub struct WebsocketConfig { pub url_runnable_args: Option>>, #[serde(default)] pub can_return_message: bool, + #[serde(default)] + pub can_return_error_result: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -48,6 +51,7 @@ pub struct WebsocketConfigRequest { initial_messages: Option>, url_runnable_args: Option, can_return_message: bool, + can_return_error_result: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -57,11 +61,11 @@ pub struct TestWebsocketConfig { } pub fn value_to_args_hashmap( - args: Option<&serde_json::Value>, + args: Option<&Box>, ) -> Result>> { let args = if let Some(args) = args { let args_map: Option> = - serde_json::from_value(args.clone()) + serde_json::from_str(args.get()) .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?; args_map @@ -85,7 +89,7 @@ pub async fn get_url_from_runnable_value( is_flow: bool, db: &DB, authed: ApiAuthed, - args: Option<&serde_json::Value>, + args: Option<&Box>, workspace_id: &str, ) -> Result { tracing::info!( @@ -96,7 +100,7 @@ pub async fn get_url_from_runnable_value( let args = value_to_args_hashmap(args)?; - let result = trigger_runnable_and_wait_for_raw_result( + let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx( db, None, authed, diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 6ded25f923..9021d6fe8e 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -71,6 +71,7 @@ pub fn workspaced_service() -> Router { .route("/get/:user", get(get_workspace_user)) .route("/update/:user", post(update_workspace_user)) .route("/delete/:user", delete(delete_workspace_user)) + .route("/convert_to_group/:user", post(convert_user_to_group)) .route("/is_owner/*path", get(is_owner_of_path)) .route("/whois/:username", get(whois)) .route("/whoami", get(whoami)) @@ -109,6 +110,7 @@ pub fn global_service() -> Router { .route("/leave_instance", post(leave_instance)) .route("/export", get(export_global_users)) .route("/overwrite", post(overwrite_global_users)) + .route("/onboarding", post(submit_onboarding_data)) // .route("/list_invite_codes", get(list_invite_codes)) // .route("/create_invite_code", post(create_invite_code)) @@ -285,6 +287,7 @@ pub struct GlobalUserInfo { username: Option, #[serde(skip_serializing_if = "Option::is_none")] operator_only: Option, + first_time_user: bool, } #[derive(Serialize, Debug)] @@ -491,14 +494,13 @@ async fn list_user_usage( UserWithUsage, " SELECT usr.email, usage.executions - FROM usr - , LATERAL ( - SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0)::BIGINT executions - FROM v2_as_completed_job - WHERE workspace_id = $1 - AND job_kind NOT IN ('flow', 'flowpreview', 'flownode') - AND email = usr.email - AND now() - '1 week'::interval < created_at + FROM usr, LATERAL ( + SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions + FROM v2_job_completed c JOIN v2_job j USING (id) + WHERE j.workspace_id = $1 + AND j.kind NOT IN ('flow', 'flowpreview', 'flownode') + AND j.permissioned_as_email = usr.email + AND now() - '1 week'::interval < j.created_at ) usage WHERE workspace_id = $1 ", @@ -532,7 +534,7 @@ async fn list_users_as_super_admin( GlobalUserInfo, "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')), authors as (SELECT distinct email FROM usr WHERE usr.operator IS false) - SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username + SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user FROM password WHERE email IN (SELECT email FROM active_users) ORDER BY super_admin DESC, devops DESC @@ -545,7 +547,7 @@ async fn list_users_as_super_admin( } else { sqlx::query_as!( GlobalUserInfo, - "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT \ + "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only, first_time_user FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT \ $1 OFFSET $2", per_page as i32, offset as i32 @@ -582,7 +584,7 @@ async fn update_tutorial_progress( Json(progress): Json, ) -> Result { sqlx::query_scalar!( - "INSERT INTO tutorial_progress VALUES ($2, $1::bigint::bit(64)) ON CONFLICT (email) DO UPDATE SET progress = $1::bigint::bit(64)", + "INSERT INTO tutorial_progress VALUES ($2, $1::bigint::bit(64)) ON CONFLICT (email) DO UPDATE SET progress = EXCLUDED.progress", progress.progress as i64, authed.email ) @@ -748,7 +750,7 @@ async fn global_whoami( ) -> JsonResult { let user = sqlx::query_as!( GlobalUserInfo, - "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only FROM password WHERE \ + "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user FROM password WHERE \ email = $1", email ) @@ -769,6 +771,7 @@ async fn global_whoami( company: None, username: None, operator_only: None, + first_time_user: false, })) } else { Err(user.unwrap_err()) @@ -1362,6 +1365,151 @@ async fn update_workspace_user( Ok(format!("user {} updated", user_email)) } +async fn convert_user_to_group( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, username_to_convert)): Path<(String, String)>, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + let mut tx = db.begin().await?; + + // Get user email and current status + let user_info = sqlx::query!( + "SELECT email, is_admin, operator, added_via FROM usr WHERE username = $1 AND workspace_id = $2", + username_to_convert, + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + + let user_info = not_found_if_none(user_info, "User", &username_to_convert)?; + + // Check if user is already a group user + if let Some(added_via) = &user_info.added_via { + if added_via.get("source").and_then(|v| v.as_str()) == Some("instance_group") { + return Err(Error::BadRequest( + "User is already a group user".to_string(), + )); + } + } + + // Find which instance groups this user belongs to that are configured for auto-add in this workspace + let eligible_groups = sqlx::query!( + r#" + SELECT + eig.igroup as group_name, + ws.auto_add_instance_groups_roles + FROM email_to_igroup eig + INNER JOIN workspace_settings ws ON ws.workspace_id = $1 + WHERE eig.email = $2 + AND eig.igroup = ANY(ws.auto_add_instance_groups) + "#, + &w_id, + &user_info.email + ) + .fetch_all(&mut *tx) + .await?; + + if eligible_groups.is_empty() { + return Err(Error::BadRequest( + "User is not a member of any instance groups configured for auto-add in this workspace" + .to_string(), + )); + } + + // Determine the group with highest precedence (same logic as process_instance_group_auto_adds) + let roles: std::collections::HashMap = + if let Some(roles_json) = &eligible_groups[0].auto_add_instance_groups_roles { + serde_json::from_value(roles_json.clone()).unwrap_or_default() + } else { + std::collections::HashMap::new() + }; + + let mut best_group = &eligible_groups[0].group_name; + let mut best_precedence = 0u8; + + for group in &eligible_groups { + let default_role = "developer".to_string(); + let role = roles.get(&group.group_name).unwrap_or(&default_role); + + let precedence = match role.as_str() { + "admin" => 3, + "developer" => 2, + "operator" => 1, + _ => 2, + }; + + if precedence > best_precedence { + best_precedence = precedence; + best_group = &group.group_name; + } + } + + let primary_group_name = best_group; + + // Determine role from group configuration using the selected primary group + let default_role = "developer".to_string(); + let role = roles + .get(primary_group_name) + .unwrap_or(&default_role) + .as_str(); + + let (is_admin, is_operator) = match role { + "admin" => (true, false), + "operator" => (false, true), + _ => (false, false), + }; + + // Update user with instance group information + let instance_group_source = serde_json::json!({ + "source": "instance_group", + "group": primary_group_name + }); + + sqlx::query!( + "UPDATE usr SET added_via = $1, is_admin = $2, operator = $3 WHERE username = $4 AND workspace_id = $5", + instance_group_source, + is_admin, + is_operator, + username_to_convert, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.convert_to_group", + ActionKind::Update, + &w_id, + Some(&username_to_convert), + Some([("group", primary_group_name.as_str()), ("role", role)].into()), + ) + .await?; + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::User { email: user_info.email.clone() }, + Some(format!( + "Converted user '{}' to group user (group: {}, role: {})", + &user_info.email, primary_group_name, role + )), + true, + ) + .await?; + + Ok(format!( + "User {} converted to group user (group: {}, role: {})", + username_to_convert, primary_group_name, role + )) +} + async fn update_user( authed: ApiAuthed, Path(email_to_update): Path, @@ -1459,6 +1607,15 @@ async fn delete_user( .execute(&mut *tx) .await?; } + + // Remove user from all instance groups email_to_igroup + sqlx::query!( + "DELETE FROM email_to_igroup WHERE email = $1", + &email_to_delete + ) + .execute(&mut *tx) + .await?; + audit_log( &mut *tx, &authed, @@ -1488,6 +1645,14 @@ async fn create_user( crate::users_oss::create_user(authed, db, webhook, argon2, nu).await } +async fn submit_onboarding_data( + authed: ApiAuthed, + Extension(db): Extension, + Json(data): Json, +) -> Result { + crate::users_oss::submit_onboarding_data(authed, Extension(db), Json(data)).await +} + /// Internal helper for updating workspace user permissions - used by both API and system operations pub async fn update_workspace_user_internal( w_id: &str, @@ -1703,15 +1868,15 @@ async fn login( username_override: None, token_prefix: None, }; - let email_w_h: Option<(String, String, bool, bool)> = sqlx::query_as( - "SELECT email, password_hash, super_admin, first_time_user FROM password WHERE email = $1 AND login_type = \ + let email_w_h: Option<(String, String, bool)> = sqlx::query_as( + "SELECT email, password_hash, super_admin FROM password WHERE email = $1 AND login_type = \ 'password'", ) .bind(&email) .fetch_optional(&mut *tx) .await?; - if let Some((email, hash, super_admin, first_time_user)) = email_w_h { + if let Some((email, hash, super_admin)) = email_w_h { let parsed_hash = PasswordHash::new(&hash).map_err(|e| Error::internal_err(e.to_string()))?; if argon2 @@ -1730,25 +1895,6 @@ async fn login( .await?; Err(Error::BadRequest("Invalid login".to_string())) } else { - if first_time_user { - sqlx::query_scalar!( - "UPDATE password SET first_time_user = false WHERE email = $1", - &email - ) - .execute(&mut *tx) - .await?; - let mut c = Cookie::new("first_time", "1"); - if let Some(domain) = COOKIE_DOMAIN.as_ref() { - c.set_domain(domain); - } - c.set_secure(false); - c.set_expires(time::OffsetDateTime::now_utc() + time::Duration::minutes(15)); - c.set_http_only(false); - c.set_path("/"); - - cookies.add(c); - } - let token = create_session_token(&email, super_admin, &mut tx, cookies).await?; let audit_author = AuditAuthor { @@ -2131,7 +2277,6 @@ struct Runnable { workspace: String, endpoint_async: String, endpoint_sync: String, - endpoint_openai_sync: String, summary: String, description: String, schema: Option, @@ -2183,10 +2328,6 @@ async fn get_all_runnables( "/w/{}/jobs/run_wait_result/f/{}", &f.workspace, &f.path ), - endpoint_openai_sync: format!( - "/w/{}/jobs/openai_sync/f/{}", - &f.workspace, &f.path - ), summary: f.summary, description: f.description, schema: f.schema, @@ -2196,8 +2337,8 @@ async fn get_all_runnables( .collect::>(), ); let scripts = sqlx::query!( - "SELECT workspace_id as workspace, path, summary, description, schema FROM script as o - WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1 AND archived = false) + "SELECT workspace_id as workspace, path, summary, description, schema FROM script as o + WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1 AND archived = false) AND workspace_id = $1 and archived = false", workspace ) .fetch_all(&mut *tx) @@ -2212,10 +2353,6 @@ async fn get_all_runnables( "/w/{}/jobs/run_wait_result/p/{}", &s.workspace, &s.path ), - endpoint_openai_sync: format!( - "/w/{}/jobs/openai_sync/p/{}", - &s.workspace, &s.path - ), summary: s.summary, description: s.description, schema: s.schema, diff --git a/backend/windmill-api/src/users_oss.rs b/backend/windmill-api/src/users_oss.rs index 88a12eb710..4d1b3976d2 100644 --- a/backend/windmill-api/src/users_oss.rs +++ b/backend/windmill-api/src/users_oss.rs @@ -10,14 +10,22 @@ use crate::db::ApiAuthed; #[cfg(not(feature = "private"))] use crate::users::{EditPassword, NewUser}; + #[cfg(not(feature = "private"))] use crate::{db::DB, webhook_util::WebhookShared}; + #[cfg(not(feature = "private"))] use argon2::Argon2; +#[cfg(not(feature = "private"))] +use axum::{extract::Extension, Json}; + #[cfg(not(feature = "private"))] use http::StatusCode; +#[cfg(not(feature = "private"))] +use serde::Deserialize; + #[cfg(not(feature = "private"))] use windmill_common::error::{Error, Result}; @@ -53,3 +61,23 @@ pub fn send_email_if_possible(_subject: &str, _content: &str, _to: &str) { "send_email_if_possible is not implemented in Windmill's Open Source repository" ); } + +#[cfg(not(feature = "private"))] +#[derive(Deserialize, Debug)] +#[allow(dead_code)] +pub struct OnboardingData { + pub touch_point: String, + pub use_case: String, +} + + +#[cfg(not(feature = "private"))] +pub async fn submit_onboarding_data( + _authed: ApiAuthed, + Extension(_db): Extension, + Json(_data): Json, +) -> Result { + Err(Error::internal_err( + "Not implemented in Windmill's Open Source repository".to_string(), + )) +} \ No newline at end of file diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index d3b820ccc0..ea31585cba 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -447,18 +447,8 @@ pub async fn acknowledge_all_critical_alerts( } #[cfg(feature = "http_trigger")] -#[derive(Clone)] -pub struct ExpiringCacheEntry { - pub value: T, - pub expiry: std::time::Instant, -} +pub use windmill_common::utils::ExpiringCacheEntry; lazy_static::lazy_static! { static ref DUCKLAKE_INSTANCE_PG_PASSWORD: std::sync::RwLock> = std::sync::RwLock::new(None); } - -#[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))] -pub async fn update_rw_lock(lock: std::sync::Arc>, value: T) -> () { - let mut w = lock.write().await; - *w = value; -} diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index cca13a7c25..07d0572e6f 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -83,6 +83,7 @@ async fn list_contextual_variables( Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), Some(chrono::offset::Utc::now()), Some(ScriptHash(1234567890)), + None, ) .await .to_vec(), @@ -210,7 +211,12 @@ async fn get_variable( } else if !value.is_empty() && decrypt_secret { let _ = tx.commit().await; let mc = build_crypt(&db, &w_id).await?; - Some(decrypt(&mc, value)?) + Some(decrypt(&mc, value).map_err(|e| { + Error::internal_err(format!( + "Error decrypting variable {}: {}", + variable.path, e + )) + })?) } else if q.include_encrypted.unwrap_or(false) { Some(value) } else { @@ -837,7 +843,12 @@ pub async fn get_value_internal<'a, 'e, A: sqlx::Acquire<'e, Database = Postgres return Err(Error::internal_err("Require oauth2 feature".to_string())); } else if !value.is_empty() { let mc = build_crypt(&db, &w_id).await?; - decrypt(&mc, value)? + decrypt(&mc, value).map_err(|e| { + Error::internal_err(format!( + "Error decrypting variable {}: {}", + variable.path, e + )) + })? } else { "".to_string() } @@ -853,34 +864,3 @@ pub async fn get_value_internal<'a, 'e, A: sqlx::Acquire<'e, Database = Postgres Ok(r) } -pub async fn get_variable_or_self(path: String, db: &DB, w_id: &str) -> Result { - if !path.starts_with("$var:") { - return Ok(path); - } - let path = path.strip_prefix("$var:").unwrap().to_string(); - - let record = sqlx::query!( - "SELECT value, is_secret - FROM variable - WHERE path = $1 AND workspace_id = $2", - &path, - &w_id - ) - .fetch_optional(db) - .await?; - - if let Some(record) = record { - let mut value = record.value; - if record.is_secret { - let mc = build_crypt(db, w_id).await?; - value = decrypt(&mc, value)?; - } - - Ok(value) - } else { - Err(Error::NotFound(format!( - "Variable not found when resolving `$var:{}`", - path - ))) - } -} diff --git a/backend/windmill-api/src/webhook_util.rs b/backend/windmill-api/src/webhook_util.rs index ecf87fcc39..362893a7a1 100644 --- a/backend/windmill-api/src/webhook_util.rs +++ b/backend/windmill-api/src/webhook_util.rs @@ -9,6 +9,7 @@ use windmill_common::METRICS_ENABLED; use crate::db::DB; use windmill_common::oauth2::InstanceEvent; +use windmill_common::utils::configure_client; #[cfg(feature = "prometheus")] lazy_static::lazy_static! { @@ -72,10 +73,10 @@ impl WebhookShared { pub fn new(mut shutdown_rx: tokio::sync::broadcast::Receiver<()>, db: DB) -> Self { let (tx, mut rx) = mpsc::unbounded_channel::(); let _process = tokio::spawn(async move { - let client = reqwest::Client::builder() + let client = configure_client(reqwest::Client::builder() .connect_timeout(Duration::from_secs(5)) // TODO: investigate pool timeouts and such if TCP load is high - .timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(5))) .build() .unwrap(); diff --git a/backend/windmill-api/src/websocket_triggers.rs b/backend/windmill-api/src/websocket_triggers.rs deleted file mode 100644 index 43e234a8c7..0000000000 --- a/backend/windmill-api/src/websocket_triggers.rs +++ /dev/null @@ -1,1005 +0,0 @@ -use anyhow::Context; - -use futures::{stream::SplitSink, SinkExt, StreamExt}; -use itertools::Itertools; -use rand::seq::SliceRandom; -use serde::{ - de::{self, MapAccess, Visitor}, - Deserialize, Deserializer, Serialize, -}; -use serde_json::{value::RawValue, Value}; -use sqlx::prelude::FromRow; -use sqlx::types::Json as SqlxJson; -use std::{collections::HashMap, fmt}; -use tokio::net::TcpStream; -use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; -use windmill_common::{ - error::{self, to_anyhow}, - triggers::TriggerKind, - utils::report_critical_error, - worker::to_raw_value, - INSTANCE_NAME, -}; -use windmill_queue::PushArgsOwned; - -use crate::{ - capture::{insert_capture_payload, WebsocketTriggerConfig}, - db::{ApiAuthed, DB}, - trigger_helpers::{trigger_runnable, trigger_runnable_and_wait_for_raw_result, TriggerJobArgs}, - users::fetch_api_authed, -}; - -use std::borrow::Cow; - -#[derive(Deserialize)] -pub struct JsonFilter { - key: String, - value: serde_json::Value, -} - -#[derive(Deserialize)] -#[serde(untagged)] -pub enum Filter { - JsonFilter(JsonFilter), -} - -#[derive(Deserialize)] -enum InitialMessage { - #[serde(rename = "raw_message")] - RawMessage(String), - #[serde(rename = "runnable_result")] - RunnableResult { path: String, args: Box, is_flow: bool }, -} - -#[derive(FromRow, Serialize, Clone)] -pub struct WebsocketTrigger { - pub workspace_id: String, - pub path: String, - pub url: String, - pub script_path: String, - pub is_flow: bool, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - #[serde(skip_serializing_if = "Option::is_none")] - pub server_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_server_ping: Option>, - pub extra_perms: serde_json::Value, - pub error: Option, - pub enabled: bool, - pub filters: Vec>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub initial_messages: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub url_runnable_args: Option>>, - pub can_return_message: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, -} - -async fn listen_to_unlistened_websockets( - db: &DB, - killpill_rx: &tokio::sync::broadcast::Receiver<()>, -) { - let websocket_triggers = sqlx::query_as::<_, WebsocketTrigger>( - r#" - SELECT - workspace_id, - path, - url, - script_path, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - filters, - initial_messages, - url_runnable_args, - can_return_message, - error_handler_path, - error_handler_args, - retry - FROM websocket_trigger - WHERE - enabled IS TRUE - AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') - "#, - ) - .fetch_all(db) - .await; - - match websocket_triggers { - Ok(mut triggers) => { - triggers.shuffle(&mut rand::rng()); - for trigger in triggers { - trigger - .maybe_listen_to_websocket(db.clone(), killpill_rx.resubscribe()) - .await; - } - } - Err(err) => { - tracing::error!("Error fetching WebSocket triggers: {:?}", err); - } - }; - - match sqlx::query_as!( - CaptureConfigForWebsocket, - r#"SELECT path, is_flow, workspace_id, trigger_config as "trigger_config!: _", owner, email FROM capture_config WHERE trigger_kind = 'websocket' AND last_client_ping > NOW() - INTERVAL '10 seconds' AND trigger_config IS NOT NULL AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')"# - ) - .fetch_all(db) - .await - { - Ok(mut captures) => { - captures.shuffle(&mut rand::rng()); - for capture in captures { - capture.maybe_listen_to_websocket(db.clone(), killpill_rx.resubscribe()).await; - } - } - Err(err) => { - tracing::error!("Error fetching capture WebSocket triggers: {:?}", err); - } - } -} - -pub fn start_websockets(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) -> () { - tokio::spawn(async move { - listen_to_unlistened_websockets(&db, &killpill_rx).await; - loop { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return; - } - _ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => { - listen_to_unlistened_websockets(&db, &killpill_rx).await; - } - } - } - }); -} - -struct SupersetVisitor<'a> { - key: &'a str, - value_to_check: &'a Value, -} - -impl<'de, 'a> Visitor<'de> for SupersetVisitor<'a> { - type Value = bool; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a JSON object with a specific key at the top level") - } - - fn visit_map(self, mut map: V) -> Result - where - V: MapAccess<'de>, - { - while let Some(key) = map.next_key::()? { - if key == self.key { - // Deserialize the value for the key and check if it's a superset - let json_value: Value = map.next_value()?; - return Ok(is_superset(&json_value, self.value_to_check)); - } else { - // Skip the value if it's not the one we're interested in - let _ = map.next_value::()?; - } - } - // If the key was not found, return false - Ok(false) - } -} - -// Function to check if json_value is a superset of value_to_check -fn is_superset(json_value: &Value, value_to_check: &Value) -> bool { - match (json_value, value_to_check) { - (Value::Object(json_map), Value::Object(check_map)) => { - // Check that all keys and values in check_map exist and match in json_map - check_map.iter().all(|(k, v)| { - json_map - .get(k) - .map_or(false, |json_val| is_superset(json_val, v)) - }) - } - (Value::Array(json_array), Value::Array(check_array)) => { - // Check that all elements in check_array exist in json_array - check_array.iter().all(|check_item| { - json_array - .iter() - .any(|json_item| is_superset(json_item, check_item)) - }) - } - _ => json_value == value_to_check, - } -} - -// A function to deserialize and check if the value at the given key is a superset of a passed value -fn is_value_superset<'a, 'de, D>( - deserializer: D, - key: &'a str, - value_to_check: &'a Value, -) -> Result -where - D: Deserializer<'de>, -{ - deserializer.deserialize_map(SupersetVisitor { key, value_to_check }) -} - -fn raw_value_to_args_hashmap( - args: Option<&Box>, -) -> error::Result>> { - let args = if let Some(args) = args { - serde_json::from_str::>>>(args.get()) - .map_err(|e| error::Error::BadRequest(format!("invalid json: {}", e)))? - .unwrap_or_else(HashMap::new) - } else { - HashMap::new() - }; - Ok(args) -} - -async fn loop_ping(db: &DB, ws: &WebsocketEnum, error: Option<&str>) -> () { - loop { - if let None = ws.update_ping(db, error).await { - return; - } - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; - } -} - -async fn get_url_from_runnable( - path: &str, - is_flow: bool, - db: &DB, - authed: ApiAuthed, - args: Option<&Box>, - workspace_id: &str, -) -> error::Result { - tracing::info!( - "Running {} {} to get WebSocket URL", - if is_flow { "flow" } else { "script" }, - path - ); - - let args = raw_value_to_args_hashmap(args)?; - - let result = trigger_runnable_and_wait_for_raw_result( - db, - None, - authed, - workspace_id, - path, - is_flow, - PushArgsOwned { args, extra: None }, - None, - None, - None, - "".to_string(), // doesn't matter as no retry/error handler - ) - .await?; - - serde_json::from_str::(result.get()).map_err(|_| { - error::Error::BadConfig(format!( - "{} {} did not return a string", - if is_flow { "Flow" } else { "Script" }, - path, - )) - }) -} - -impl WebsocketTrigger { - async fn maybe_listen_to_websocket( - self, - db: DB, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> () { - match sqlx::query_scalar!( - "UPDATE websocket_trigger SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE enabled IS TRUE AND workspace_id = $2 AND path = $3 AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - *INSTANCE_NAME, - self.workspace_id, - self.path, - ).fetch_optional(&db).await { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tokio::spawn(listen_to_websocket(WebsocketEnum::Trigger(self), db, killpill_rx)); - } else { - tracing::info!("WebSocket {} already being listened to", self.url); - } - }, - Err(err) => { - tracing::error!("Error acquiring lock for WebSocket {}: {:?}", self.path, err); - } - }; - } - - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - match sqlx::query_scalar!( - "UPDATE websocket_trigger SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND server_id = $4 AND enabled IS TRUE RETURNING 1", - error, - self.workspace_id, - self.path, - *INSTANCE_NAME - ).fetch_optional(db).await { - Ok(updated) => { - if updated.flatten().is_none() { - // allow faster restart of websocket trigger - sqlx::query!( - "UPDATE websocket_trigger SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND server_id IS NULL", - self.workspace_id, - self.path, - ).execute(db).await.ok(); - tracing::info!("WebSocket {} changed, disabled, or deleted, stopping...", self.url); - return None; - } - }, - Err(err) => { - tracing::warn!("Error updating ping of WebSocket {}: {:?}", self.url, err); - } - }; - - Some(()) - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - match sqlx::query!( - "UPDATE websocket_trigger SET enabled = FALSE, error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3", - error, - self.workspace_id, - self.path, - ) - .execute(db).await { - Ok(_) => { - report_critical_error(format!("Disabling WebSocket {} because of error: {}", self.url, error), db.clone(), Some(&self.workspace_id), None).await; - }, - Err(disable_err) => { - report_critical_error( - format!("Could not disable WebSocket {} with err {}, disabling because of error {}", self.path, disable_err, error), - db.clone(), - Some(&self.workspace_id), - None, - ).await; - } - } - } - - async fn get_url_from_runnable( - &self, - path: &str, - is_flow: bool, - db: &DB, - ) -> error::Result { - get_url_from_runnable( - &path, - is_flow, - db, - self.fetch_authed(db).await?, - self.url_runnable_args.as_ref().map(|r| &r.0), - &self.workspace_id, - ) - .await - } - - async fn send_initial_messages( - &self, - writer: &mut SplitSink>, Message>, - db: &DB, - ) -> error::Result<()> { - let initial_messages: Vec = self - .initial_messages - .as_deref() - .unwrap_or_default() - .iter() - .filter_map(|m| serde_json::from_str(m.get()).ok()) - .collect_vec(); - - let mut authed_o = None; - for start_message in initial_messages { - match start_message { - InitialMessage::RawMessage(msg) => { - let msg = if msg.starts_with("\"") && msg.ends_with("\"") { - msg[1..msg.len() - 1].to_string() - } else { - msg - }; - tracing::info!( - "Sending raw message initial message to WebSocket {}: {}", - self.url, - msg - ); - writer - .send(tokio_tungstenite::tungstenite::Message::Text(msg)) - .await - .map_err(to_anyhow) - .with_context(|| "failed to send raw message")?; - } - InitialMessage::RunnableResult { path, is_flow, args } => { - tracing::info!( - "Running {} {} for initial message to WebSocket {}", - if is_flow { "flow" } else { "script" }, - path, - self.url, - ); - - let args = raw_value_to_args_hashmap(Some(&args))?; - - if authed_o.is_none() { - authed_o = Some(self.fetch_authed(db).await?); - } - let authed = authed_o.clone().unwrap(); - - let result = trigger_runnable_and_wait_for_raw_result( - db, - None, - authed.clone(), - &self.workspace_id, - &path, - is_flow, - PushArgsOwned { args, extra: None }, - None, - None, - None, - "".to_string(), // doesn't matter as no retry/error handler - ) - .await - .map(|r| r.get().to_owned())?; - - tracing::info!( - "Sending {} {} result to WebSocket {}", - if is_flow { "flow" } else { "script" }, - path, - self.url - ); - - // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. - // it falls back to the original serialized JSON if it doesn't work. - let result = serde_json::from_str::(result.as_str()).unwrap_or(result); - - writer - .send(tokio_tungstenite::tungstenite::Message::Text(result)) - .await - .map_err(to_anyhow) - .with_context(|| { - format!( - "Failed to send {} {} result", - if is_flow { "flow" } else { "script" }, - path - ) - })?; - } - } - } - - Ok(()) - } - - async fn handle( - &self, - db: &DB, - msg: &str, - trigger_info: HashMap>, - return_message_channels: Option, - ) -> () { - if let Err(err) = run_job(db, self, &msg, trigger_info, return_message_channels).await { - report_critical_error( - format!( - "Failed to trigger job from WebSocket {}: {:?}", - self.url, err - ), - db.clone(), - Some(&self.workspace_id), - None, - ) - .await; - }; - } - - async fn fetch_authed(&self, db: &DB) -> error::Result { - fetch_api_authed( - self.edited_by.clone(), - self.email.clone(), - &self.workspace_id, - db, - Some(format!("ws-{}", self.path)), - ) - .await - } -} - -impl TriggerJobArgs<&str> for WebsocketTrigger { - fn v1_payload_fn(payload: &str) -> HashMap> { - HashMap::from([("msg".to_string(), to_raw_value(&payload))]) - } - - fn trigger_kind() -> TriggerKind { - TriggerKind::Websocket - } -} - -#[derive(Deserialize)] -struct CaptureConfigForWebsocket { - trigger_config: SqlxJson, - path: String, - is_flow: bool, - workspace_id: String, - owner: String, - email: String, -} - -impl CaptureConfigForWebsocket { - async fn maybe_listen_to_websocket( - self, - db: DB, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> () { - match sqlx::query_scalar!( - "UPDATE capture_config SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE last_client_ping > NOW() - INTERVAL '10 seconds' AND workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket' AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - *INSTANCE_NAME, - self.workspace_id, - self.path, - self.is_flow, - ).fetch_optional(&db).await { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tokio::spawn(listen_to_websocket(WebsocketEnum::Capture(self), db, killpill_rx)); - } else { - tracing::info!("WebSocket {} already being listened to", self.trigger_config.url); - } - }, - Err(err) => { - tracing::error!("Error acquiring lock for capture WebSocket {}: {:?}", self.path, err); - } - }; - } - - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - match sqlx::query_scalar!( - "UPDATE capture_config SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket' AND server_id = $5 AND last_client_ping > NOW() - INTERVAL '10 seconds' RETURNING 1", - error, - self.workspace_id, - self.path, - self.is_flow, - *INSTANCE_NAME - ).fetch_optional(db).await { - Ok(updated) => { - if updated.flatten().is_none() { - // allow faster restart of websocket capture - sqlx::query!( - "UPDATE capture_config SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = 'websocket' AND server_id IS NULL", - self.workspace_id, - self.path, - self.is_flow, - ).execute(db).await.ok(); - tracing::info!("WebSocket capture {} changed, disabled, or deleted, stopping...", self.trigger_config.url); - return None; - } - }, - Err(err) => { - tracing::warn!("Error updating ping of capture WebSocket {}: {:?}", self.trigger_config.url, err); - } - }; - - Some(()) - } - - async fn handle(&self, db: &DB, msg: &str, trigger_info: HashMap>) -> () { - let (main_args, preprocessor_args) = - WebsocketTrigger::build_capture_payloads(&msg, trigger_info); - - if let Err(err) = insert_capture_payload( - db, - &self.workspace_id, - &self.path, - self.is_flow, - &TriggerKind::Websocket, - main_args, - preprocessor_args, - &self.owner, - ) - .await - { - tracing::error!("Error inserting capture payload: {:?}", err); - } - } - - async fn get_url_from_runnable( - &self, - path: &str, - is_flow: bool, - db: &DB, - ) -> error::Result { - let url_runnable_args = self - .trigger_config - .url_runnable_args - .as_ref() - .map(to_raw_value); - get_url_from_runnable( - &path, - is_flow, - db, - self.fetch_authed(db).await?, - url_runnable_args.as_ref(), - &self.workspace_id, - ) - .await - } - - async fn fetch_authed(&self, db: &DB) -> error::Result { - fetch_api_authed( - self.owner.clone(), - self.email.clone(), - &self.workspace_id, - db, - Some(format!("ws-{}", self.get_trigger_path())), - ) - .await - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - if let Err(err) = sqlx::query!( - "UPDATE capture_config SET error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket'", - error, - self.workspace_id, - self.path, - self.is_flow, - ) - .execute(db).await { - tracing::error!("Could not disable WebSocket capture {} ({}) with err {}, disabling because of error {}", self.path, self.workspace_id, err, error); - } - } - - fn get_trigger_path(&self) -> String { - format!( - "{}-{}", - if self.is_flow { "flow" } else { "script" }, - self.path - ) - } -} - -enum WebsocketEnum { - Trigger(WebsocketTrigger), - Capture(CaptureConfigForWebsocket), -} - -impl WebsocketEnum { - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - match self { - WebsocketEnum::Trigger(ws) => ws.update_ping(db, error).await, - WebsocketEnum::Capture(capture) => capture.update_ping(db, error).await, - } - } - - async fn get_url_from_runnable( - &self, - path: &str, - is_flow: bool, - db: &DB, - ) -> error::Result { - match self { - WebsocketEnum::Trigger(ws) => ws.get_url_from_runnable(path, is_flow, db).await, - WebsocketEnum::Capture(capture) => { - capture.get_url_from_runnable(path, is_flow, db).await - } - } - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - match self { - WebsocketEnum::Trigger(ws) => ws.disable_with_error(db, error).await, - WebsocketEnum::Capture(capture) => capture.disable_with_error(db, error).await, - } - } -} - -struct ReturnMessageChannels { - send_message_tx: tokio::sync::mpsc::Sender, - killpill_rx: tokio::sync::broadcast::Receiver<()>, -} - -impl Clone for ReturnMessageChannels { - fn clone(&self) -> Self { - Self { - send_message_tx: self.send_message_tx.clone(), - killpill_rx: self.killpill_rx.resubscribe(), - } - } -} - -async fn listen_to_websocket( - ws: WebsocketEnum, - db: DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) -> () { - let url = match &ws { - WebsocketEnum::Trigger(ws_trigger) => ws_trigger.url.clone(), - WebsocketEnum::Capture(capture) => capture.trigger_config.url.clone(), - }; - - let filters: Vec = match &ws { - WebsocketEnum::Trigger(ws_trigger) => ws_trigger - .filters - .iter() - .filter_map(|m| serde_json::from_str(m.get()).ok()) - .collect_vec(), - WebsocketEnum::Capture(_) => vec![], - }; - - let connect_url: Cow = if url.starts_with("$") { - if url.starts_with("$flow:") || url.starts_with("$script:") { - let path = url.splitn(2, ':').nth(1).unwrap(); - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return; - }, - _ = loop_ping(&db, &ws, Some( - "Waiting on runnable to return WebSocket URL..." - )) => { - return; - }, - - url_result = ws.get_url_from_runnable(path, url.starts_with("$flow:"), &db) => match url_result { - Ok(url) => Cow::Owned(url), - Err(err) => { - ws.disable_with_error(&db, format!( - "Error getting WebSocket URL from runnable after 5 tries: {:?}", - err - ), - ) - .await; - return; - } - }, - } - } else { - ws.disable_with_error(&db, format!("Invalid WebSocket runnable path: {}", url)) - .await; - return; - } - } else { - Cow::Borrowed(&url) - }; - - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return; - }, - _ = loop_ping(&db, &ws, Some("Connecting...")) => { - return; - }, - connection = connect_async(connect_url.as_ref()) => { - match connection { - Ok((ws_stream, _)) => { - tracing::info!("Connected to WebSocket {}", url); - let (mut writer, mut reader) = ws_stream.split(); - - // send initial messages - match &ws { - WebsocketEnum::Trigger(ws_trigger) => { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return; - }, - _ = loop_ping(&db, &ws, Some("Sending initial messages...")) => { - return; - }, - result = ws_trigger.send_initial_messages(&mut writer, &db) => { - if let Err(err) = result { - ws_trigger.disable_with_error(&db, format!("Error sending initial messages: {:?}", err)).await; - return - } else { - tracing::debug!("Initial messages sent successfully to WebSocket {}", url); - } - } - } - }, - _ => { - } - } - - let (return_message_channels, message_sender_handle) = match &ws { - WebsocketEnum::Trigger(ws_trigger) if ws_trigger.can_return_message => { - let (send_message_tx, mut rx) = tokio::sync::mpsc::channel::(100); - let w_id = ws_trigger.workspace_id.clone(); - let url = ws_trigger.url.clone(); - let db = db.clone(); - let handle = tokio::spawn(async move { - while let Some(message) = rx.recv().await { - if let Err(err) = writer.send(tokio_tungstenite::tungstenite::Message::Text(message)).await { - report_critical_error(format!("Could not send runnable result to WebSocket {} because of error: {}", url, err), db.clone(), Some(&w_id), None).await; - } - } - }); - - let killpill_rx = killpill_rx.resubscribe(); - - let return_message_channels = ReturnMessageChannels { - send_message_tx, - killpill_rx - }; - - (Some(return_message_channels), Some(handle)) - }, - _ => (None, None) - }; - - tokio::select! { - biased; - _ = killpill_rx.recv() => {}, - _ = loop_ping(&db, &ws, None) => {}, - _ = async { - loop { - if let Some(msg) = reader.next().await { - match msg { - Ok(msg) => { - match msg { - tokio_tungstenite::tungstenite::Message::Text(text) => { - tracing::debug!("Received text message from WebSocket {}: {}", url, text); - let mut should_handle = true; - for filter in &filters { - match filter { - Filter::JsonFilter(JsonFilter { key, value }) => { - let mut deserializer = serde_json::Deserializer::from_str(text.as_str()); - should_handle = match is_value_superset(&mut deserializer, key, &value) { - Ok(filter_match) => { - filter_match - }, - Err(err) => { - tracing::warn!("Error deserializing filter for WebSocket {}: {:?}", url, err); - false - } - }; - } - } - if !should_handle { - break; - } - } - if should_handle { - let trigger_info = HashMap::from([ - ("url".to_string(), to_raw_value(&url)), - ]); - match &ws { - WebsocketEnum::Trigger(ws_trigger) => { - ws_trigger.handle(&db, &text, trigger_info, return_message_channels.clone()).await; - }, - WebsocketEnum::Capture(capture) => { - capture.handle(&db, &text, trigger_info).await; - }, - } - } - }, - a @ _ => { - tracing::debug!("Received non text-message from WebSocket {}: {:?}", url, a); - } - } - }, - Err(err) => { - tracing::error!("Error reading from WebSocket {}: {:?}", url, err); - } - } - } else { - tracing::error!("WebSocket {} closed", url); - ws.update_ping(&db, Some("WebSocket closed")).await; - break; - } - } - } => {} - } - // make sure to stop return message handler - if let Some(message_sender_handle) = message_sender_handle { - message_sender_handle.abort(); - } - } - Err(err) => { - tracing::error!("Error connecting to WebSocket {}: {:?}", url, err); - ws.update_ping(&db, Some(err.to_string().as_str())).await; - } - } - } - } -} - -async fn run_job( - db: &DB, - trigger: &WebsocketTrigger, - msg: &str, - trigger_info: HashMap>, - return_message_channels: Option, -) -> anyhow::Result<()> { - let args = WebsocketTrigger::build_job_args( - &trigger.script_path, - trigger.is_flow, - &trigger.workspace_id, - db, - msg, - trigger_info, - ) - .await?; - - let authed = fetch_api_authed( - trigger.edited_by.clone(), - trigger.email.clone(), - &trigger.workspace_id, - db, - Some(format!("ws-{}", trigger.path)), - ) - .await?; - - if let Some(ReturnMessageChannels { send_message_tx, mut killpill_rx }) = - return_message_channels - { - let db_ = db.clone(); - let url = trigger.url.clone(); - let script_path = trigger.script_path.clone(); - let is_flow = trigger.is_flow; - let w_id = trigger.workspace_id.clone(); - let retry = trigger.retry.clone(); - let error_handler_path = trigger.error_handler_path.clone(); - let error_handler_args = trigger.error_handler_args.clone(); - let trigger_path = trigger.path.clone(); - let handle_response_f = async move { - tokio::select! { - _ = killpill_rx.recv() => { - return; - }, - result = trigger_runnable_and_wait_for_raw_result( - &db_, - None, - authed, - &w_id, - &script_path, - is_flow, - args, - retry.as_ref(), - error_handler_path.as_deref(), - error_handler_args.as_ref(), - format!("websocket_trigger/{}", trigger_path), - ) => { - if let Ok(result) = result.map(|r| r.get().to_owned()) { - // only send the result if it's not null - if result != "null" { - tracing::info!("Sending job result to WebSocket {}", url); - // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. - // it falls back to the original serialized JSON if it doesn't work. - let result = serde_json::from_str::(result.as_str()).unwrap_or(result); - if let Err(err) = send_message_tx.send(result).await { - report_critical_error(format!("Could not send runnable result to WebSocket {} because of error: {}", url, err), db_.clone(), Some(&w_id), None).await; - } - } - } - } - }; - }; - - tokio::spawn(handle_response_f); - } else { - trigger_runnable( - db, - None, - authed, - &trigger.workspace_id, - &trigger.script_path, - trigger.is_flow, - args, - trigger.retry.as_ref(), - trigger.error_handler_path.as_deref(), - trigger.error_handler_args.as_ref(), - format!("websocket_trigger/{}", trigger.path), - ) - .await?; - } - - Ok(()) -} diff --git a/backend/windmill-api/src/workers.rs b/backend/windmill-api/src/workers.rs index 1165a5e2f3..4e6c5ab086 100644 --- a/backend/windmill-api/src/workers.rs +++ b/backend/windmill-api/src/workers.rs @@ -71,10 +71,10 @@ struct WorkerPing { wm_memory_usage: Option, } -#[derive(Serialize, Deserialize)] -struct EnableWorkerQuery { - disable: bool, -} +// #[derive(Serialize, Deserialize)] +// struct EnableWorkerQuery { +// disable: bool, +// } #[derive(Deserialize)] pub struct ListWorkerQuery { diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index c7dad1d74d..29abc097f8 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -29,16 +29,13 @@ use regex::Regex; use hex; use sha2::{Digest, Sha256}; -use std::collections::{hash_map::DefaultHasher, HashMap}; -use std::hash::{Hash, Hasher}; +use std::collections::HashMap; use uuid::Uuid; use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::db::UserDB; use windmill_common::s3_helpers::LargeFileStorage; -use windmill_common::scripts::{NewScript, ScriptKind, ScriptLang}; use windmill_common::users::username_to_permissioned_as; -use windmill_common::variables::ExportableListableVariable; use windmill_common::variables::{build_crypt, decrypt, encrypt, WORKSPACE_CRYPT_CACHE}; use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; #[cfg(feature = "enterprise")] @@ -54,6 +51,7 @@ use windmill_common::{ utils::{paginate, rd_string, require_admin, Pagination}, }; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; +use windmill_worker::scoped_dependency_map::{DependencyMap, ScopedDependencyMap}; #[cfg(feature = "enterprise")] use windmill_common::utils::require_admin_or_devops; @@ -81,6 +79,8 @@ pub fn workspaced_service() -> Router { .route("/invite_user", post(invite_user)) .route("/add_user", post(add_user)) .route("/delete_invite", post(delete_invite)) + .route("/rebuild_dependency_map", post(rebuild_dependency_map)) + .route("/get_dependency_map", get(get_dependency_map)) .route("/get_settings", get(get_settings)) .route("/get_deploy_to", get(get_deploy_to)) .route("/edit_slack_command", post(edit_slack_command)) @@ -102,6 +102,9 @@ pub fn workspaced_service() -> Router { "/run_teams_message_test_job", post(run_teams_message_test_job), ) + .route("/slack_oauth_config", get(get_slack_oauth_config)) + .route("/slack_oauth_config", post(set_slack_oauth_config)) + .route("/slack_oauth_config", delete(delete_slack_oauth_config)) .route("/edit_webhook", post(edit_webhook)) .route("/edit_auto_invite", post(edit_auto_invite)) .route("/edit_instance_groups", post(edit_instance_groups)) @@ -141,6 +144,7 @@ pub fn workspaced_service() -> Router { ) .route("/leave", post(leave_workspace)) .route("/get_workspace_name", get(get_workspace_name)) + .route("/create_fork", post(create_workspace_fork)) .route("/change_workspace_name", post(change_workspace_name)) .route("/change_workspace_color", post(change_workspace_color)) .route( @@ -176,7 +180,7 @@ pub fn global_service() -> Router { .route("/list", get(list_workspaces)) .route("/users", get(user_workspaces)) .route("/create", post(create_workspace)) - .route("/create_fork", post(create_workspace_fork)) + .route("/create_fork", post(deprecated_create_workspace_fork)) .route("/exists", post(exists_workspace)) .route("/exists_username", post(exists_username)) .route("/allowed_domain_auto_invite", get(is_allowed_auto_domain)) @@ -218,6 +222,10 @@ pub struct WorkspaceSettings { pub teams_command_script: Option, pub slack_email: String, #[serde(skip_serializing_if = "Option::is_none")] + pub slack_oauth_client_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub slack_oauth_client_secret: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub auto_invite_domain: Option, #[serde(skip_serializing_if = "Option::is_none")] pub auto_invite_operator: Option, @@ -265,11 +273,11 @@ pub struct WorkspaceSettings { pub auto_add_instance_groups_roles: Option, } -#[derive(sqlx::Type, Serialize, Deserialize, Debug)] -#[sqlx(type_name = "WORKSPACE_KEY_KIND", rename_all = "lowercase")] -pub enum WorkspaceKeyKind { - Cloud, -} +// #[derive(sqlx::Type, Serialize, Deserialize, Debug)] +// #[sqlx(type_name = "WORKSPACE_KEY_KIND", rename_all = "lowercase")] +// pub enum WorkspaceKeyKind { +// Cloud, +// } #[derive(Deserialize)] struct EditCommandScript { @@ -342,9 +350,7 @@ struct CreateWorkspace { struct CreateWorkspaceFork { id: String, name: String, - username: Option, color: Option, - parent_workspace_id: String, } #[derive(Deserialize)] @@ -367,6 +373,7 @@ struct UserWorkspace { pub color: Option, pub operator_settings: Option>, pub parent_workspace_id: Option, + pub disabled: bool, } #[derive(Deserialize)] @@ -439,7 +446,7 @@ async fn is_premium( require_admin(authed.is_admin, &authed.username)?; #[cfg(feature = "cloud")] let premium = windmill_common::workspaces::get_team_plan_status(&_db, &_w_id) - .await + .await? .premium; #[cfg(not(feature = "cloud"))] let premium = false; @@ -501,6 +508,8 @@ async fn get_settings( slack_command_script, teams_command_script, slack_email, + slack_oauth_client_id, + slack_oauth_client_secret, auto_invite_domain, auto_invite_operator, auto_add, @@ -669,6 +678,122 @@ async fn run_slack_message_test_job( })) } +#[derive(Deserialize)] +struct SetSlackOAuthConfigRequest { + slack_oauth_client_id: String, + slack_oauth_client_secret: String, +} + +#[derive(Serialize)] +struct GetSlackOAuthConfigResponse { + slack_oauth_client_id: Option, + slack_oauth_client_secret: Option, +} + +async fn get_slack_oauth_config( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult { + require_admin(authed.is_admin, &authed.username)?; + + let settings = sqlx::query_as!( + WorkspaceSettings, + "SELECT * FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await?; + + // Mask the secret if it exists + let masked_secret = settings.slack_oauth_client_secret.map(|_| "***".to_string()); + + Ok(Json(GetSlackOAuthConfigResponse { + slack_oauth_client_id: settings.slack_oauth_client_id, + slack_oauth_client_secret: masked_secret, + })) +} + +async fn set_slack_oauth_config( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + if req.slack_oauth_client_id.is_empty() || req.slack_oauth_client_secret.is_empty() { + return Err(Error::BadRequest( + "Both client ID and client secret are required".to_string(), + )); + } + + let mut tx = db.begin().await?; + + sqlx::query!( + "UPDATE workspace_settings + SET slack_oauth_client_id = $1, slack_oauth_client_secret = $2 + WHERE workspace_id = $3", + &req.slack_oauth_client_id, + &req.slack_oauth_client_secret, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.set_slack_oauth_config", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some([("client_id", req.slack_oauth_client_id.as_str())].into()), + ) + .await?; + + tx.commit().await?; + + Ok(format!("Slack OAuth config set for workspace {}", &w_id)) +} + +async fn delete_slack_oauth_config( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = db.begin().await?; + + sqlx::query!( + "UPDATE workspace_settings + SET slack_oauth_client_id = NULL, slack_oauth_client_secret = NULL + WHERE workspace_id = $1", + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.delete_slack_oauth_config", + ActionKind::Delete, + &w_id, + Some(&authed.email), + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!( + "Slack OAuth config deleted for workspace {}", + &w_id + )) +} + async fn get_secondary_storage_names( _authed: ApiAuthed, Extension(db): Extension, @@ -927,6 +1052,7 @@ async fn get_copilot_info( default_model: None, code_completion_model: None, custom_prompts: None, + max_tokens_per_model: None, })) } } @@ -1022,10 +1148,11 @@ async fn edit_ducklake_config( authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, - ApiAuthed { is_admin, username, .. }: ApiAuthed, + ApiAuthed { is_admin, username, email, .. }: ApiAuthed, Json(new_config): Json, ) -> Result { require_admin(is_admin, &username)?; + let is_superadmin = require_super_admin(&db, &email).await.is_ok(); let mut tx = db.begin().await?; @@ -1064,6 +1191,38 @@ async fn edit_ducklake_config( } } + // Check that non-superadmins are not abusing Instance catalogs + if !is_superadmin { + let old_ducklakes = sqlx::query_scalar!( + r#" + SELECT ws.ducklake->'ducklakes' AS ducklake_name + FROM workspace_settings ws + WHERE ws.workspace_id = $1 + "#, + &w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(serde_json::Value::Null); + let old_ducklakes: HashMap = + serde_json::from_value(old_ducklakes).unwrap_or_default(); + for (name, dl) in new_config.settings.ducklakes.iter() { + if dl.catalog.resource_type == DucklakeCatalogResourceType::Instance { + let old_dl = old_ducklakes.get(name); + if old_dl.is_none() + || old_dl.unwrap().catalog.resource_type + != DucklakeCatalogResourceType::Instance + || old_dl.unwrap().catalog.resource_path != dl.catalog.resource_path + { + return Err(Error::BadRequest( + "Only superadmins can create or modify ducklakes with Instance catalogs" + .to_string(), + )); + } + } + } + } + let config: serde_json::Value = serde_json::to_value(new_config.settings) .map_err(|err| Error::internal_err(err.to_string()))?; @@ -1491,9 +1650,9 @@ async fn delete_git_sync_repository( )) } +#[cfg(feature = "enterprise")] #[derive(Debug, Deserialize)] struct EditDeployUIConfig { - #[cfg(feature = "enterprise")] deploy_ui_settings: Option, } @@ -1811,7 +1970,7 @@ async fn edit_error_handler( SET error_handler = NULL, error_handler_extra_args = NULL, - error_handler_muted_on_cancel = NULL + error_handler_muted_on_cancel = false WHERE workspace_id = $1 "#, @@ -1867,7 +2026,7 @@ async fn set_environment_variable( match value { Some(value) => { sqlx::query!( - "INSERT INTO workspace_env (workspace_id, name, value) VALUES ($1, $2, $3) ON CONFLICT (workspace_id, name) DO UPDATE SET value = $3", + "INSERT INTO workspace_env (workspace_id, name, value) VALUES ($1, $2, $3) ON CONFLICT (workspace_id, name) DO UPDATE SET value = EXCLUDED.value", &w_id, name, value @@ -2109,7 +2268,8 @@ async fn user_workspaces( let workspaces = sqlx::query_as!( UserWorkspace, "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id, - CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings + CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings, + usr.disabled FROM workspace JOIN usr ON usr.workspace_id = workspace.id JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id @@ -2341,105 +2501,49 @@ async fn create_workspace( Ok(format!("Created workspace {}", &nw.id)) } -fn hash_script(ns: &NewScript) -> i64 { - let mut dh = DefaultHasher::new(); - ns.hash(&mut dh); - dh.finish() as i64 -} - async fn clone_workspace_data( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, target_workspace_id: &str, - target_username: &str, - db: &DB, ) -> Result<()> { // Clone workspace settings (merge with existing basic settings) - update_workspace_settings( - tx, - source_workspace_id, - target_workspace_id, - target_username, - ) - .await?; + update_workspace_settings(tx, source_workspace_id, target_workspace_id).await?; // Clone workspace environment variables clone_workspace_env(tx, source_workspace_id, target_workspace_id).await?; // Clone folders - clone_folders( - tx, - source_workspace_id, - target_workspace_id, - target_username, - ) - .await?; + clone_folders(tx, source_workspace_id, target_workspace_id).await?; // Clone groups clone_groups(tx, source_workspace_id, target_workspace_id).await?; // Clone resource types - clone_resource_types( - tx, - source_workspace_id, - target_workspace_id, - target_username, - ) - .await?; + clone_resource_types(tx, source_workspace_id, target_workspace_id).await?; // Clone resources - clone_resources( - tx, - source_workspace_id, - target_workspace_id, - target_username, - ) - .await?; + clone_resources(tx, source_workspace_id, target_workspace_id).await?; // Clone variables with re-encryption - clone_variables(tx, source_workspace_id, target_workspace_id, db).await?; + clone_variables(tx, source_workspace_id, target_workspace_id).await?; // Clone scripts with new hashes - let script_hash_mapping = clone_scripts( - tx, - source_workspace_id, - target_workspace_id, - target_username, - ) - .await?; + clone_scripts(tx, source_workspace_id, target_workspace_id).await?; // Clone flows with new versions - clone_flows( - tx, - source_workspace_id, - target_workspace_id, - target_username, - ) - .await?; + clone_flows(tx, source_workspace_id, target_workspace_id).await?; // Clone flow nodes clone_flow_nodes(tx, source_workspace_id, target_workspace_id).await?; // Clone apps with new IDs and app scripts - let _app_id_mapping = clone_apps( - tx, - source_workspace_id, - target_workspace_id, - target_username, - ) - .await?; + let _app_id_mapping = clone_apps(tx, source_workspace_id, target_workspace_id).await?; // Clone raw apps clone_raw_apps(tx, source_workspace_id, target_workspace_id).await?; - // Clone workspace runnable dependencies with updated mappings - clone_workspace_dependencies( - tx, - source_workspace_id, - target_workspace_id, - &script_hash_mapping, - ) - .await?; + // Clone workspace runnable dependencies and dependency map + clone_workspace_dependencies(tx, source_workspace_id, target_workspace_id).await?; Ok(()) } @@ -2448,8 +2552,16 @@ async fn update_workspace_settings( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, target_workspace_id: &str, - _target_username: &str, ) -> Result<()> { + sqlx::query!( + "INSERT INTO workspace_key (workspace_id, kind, key) + SELECT $2, kind, key FROM workspace_key WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + sqlx::query!( r#" UPDATE workspace_settings @@ -2527,16 +2639,14 @@ async fn clone_folders( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, target_workspace_id: &str, - target_username: &str, ) -> Result<()> { sqlx::query!( "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, edited_at, created_by) - SELECT $2, name, display_name, owners, extra_perms, summary, edited_at, $3 + SELECT $2, name, display_name, owners, extra_perms, summary, edited_at, created_by FROM folder WHERE workspace_id = $1", source_workspace_id, target_workspace_id, - target_username, ) .execute(&mut **tx) .await?; @@ -2567,16 +2677,14 @@ async fn clone_resource_types( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, target_workspace_id: &str, - target_username: &str, ) -> Result<()> { sqlx::query!( "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension) - SELECT $2, name, schema, description, edited_at, $3, format_extension - FROM resource_type + SELECT $2, name, schema, description, edited_at, created_by, format_extension + FROM resource_type WHERE workspace_id = $1", source_workspace_id, target_workspace_id, - target_username, ) .execute(&mut **tx) .await?; @@ -2588,16 +2696,14 @@ async fn clone_resources( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, target_workspace_id: &str, - target_username: &str, ) -> Result<()> { sqlx::query!( "INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, edited_at, created_by) - SELECT $2, path, value, description, resource_type, extra_perms, edited_at, $3 - FROM resource + SELECT $2, path, value, description, resource_type, extra_perms, edited_at, created_by + FROM resource WHERE workspace_id = $1", source_workspace_id, target_workspace_id, - target_username, ) .execute(&mut **tx) .await?; @@ -2609,81 +2715,18 @@ async fn clone_variables( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, target_workspace_id: &str, - db: &DB, ) -> Result<()> { - // Get all variables from source workspace - let variables = sqlx::query_as!( - ExportableListableVariable, - "SELECT workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at - FROM variable + sqlx::query!( + "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at) + SELECT $2, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at + FROM variable WHERE workspace_id = $1", - source_workspace_id + source_workspace_id, + target_workspace_id, ) - .fetch_all(&mut **tx) + .execute(&mut **tx) .await?; - if variables.is_empty() { - return Ok(()); - } - - // Get workspace keys from within the transaction - let source_key = sqlx::query_scalar!( - "SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud'", - source_workspace_id - ) - .fetch_one(db) - .await?; - - let target_key = sqlx::query_scalar!( - "SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud'", - target_workspace_id - ) - .fetch_one(&mut **tx) - .await?; - - // Build encryption keys manually - use windmill_common::variables::SECRET_SALT; - let source_crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { - format!("{}{}", source_key, salt) - } else { - source_key - }; - let target_crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { - format!("{}{}", target_key, salt) - } else { - target_key - }; - - let source_mc = magic_crypt::new_magic_crypt!(source_crypt_key, 256); - let target_mc = magic_crypt::new_magic_crypt!(target_crypt_key, 256); - - // Process each variable - for var in variables { - let final_value = if var.is_secret && var.value.is_some() { - // Decrypt with source key and re-encrypt with target key - let decrypted_value = decrypt(&source_mc, var.value.unwrap())?; - Some(encrypt(&target_mc, &decrypted_value)) - } else { - var.value - }; - - sqlx::query!( - "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", - target_workspace_id, - var.path, - final_value, - var.is_secret, - var.description, - var.extra_perms, - var.account, - var.is_oauth, - var.expires_at, - ) - .execute(&mut **tx) - .await?; - } - Ok(()) } @@ -2691,152 +2734,43 @@ async fn clone_scripts( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, target_workspace_id: &str, - target_username: &str, -) -> Result> { - // Get all scripts from source workspace - let scripts = sqlx::query!( - r#"SELECT hash, path, summary, description, content, - created_at, archived, schema, deleted, is_template, - extra_perms, lock, lock_error_logs, language as "language: ScriptLang", - kind as "kind: ScriptKind", tag, draft_only, envs, concurrent_limit, - concurrency_time_window_s, cache_ttl, dedicated_worker, - ws_error_handler_muted, priority, timeout, delete_after_use, - restart_unless_cancelled, concurrency_key, visible_to_runner_only, - no_main_func, codebase, has_preprocessor, on_behalf_of_email, - parent_hashes, assets - FROM script WHERE workspace_id = $1"#, +) -> Result<()> { + // Clone all scripts directly with a single query + sqlx::query!( + r#"INSERT INTO script ( + workspace_id, hash, path, parent_hashes, summary, description, content, + created_by, created_at, archived, schema, deleted, is_template, + extra_perms, lock, lock_error_logs, language, kind, tag, draft_only, + envs, concurrent_limit, concurrency_time_window_s, cache_ttl, + dedicated_worker, ws_error_handler_muted, priority, timeout, + delete_after_use, restart_unless_cancelled, concurrency_key, + visible_to_runner_only, no_main_func, codebase, has_preprocessor, + on_behalf_of_email, assets + ) + SELECT + $1, hash, path, parent_hashes, summary, description, content, + created_by, created_at, archived, schema, deleted, is_template, + extra_perms, lock, lock_error_logs, language, kind, tag, draft_only, + envs, concurrent_limit, concurrency_time_window_s, cache_ttl, + dedicated_worker, ws_error_handler_muted, priority, timeout, + delete_after_use, restart_unless_cancelled, concurrency_key, + visible_to_runner_only, no_main_func, codebase, has_preprocessor, + on_behalf_of_email, assets + FROM script + WHERE workspace_id = $2"#, + target_workspace_id, source_workspace_id ) - .fetch_all(&mut **tx) + .execute(&mut **tx) .await?; - let mut script_hash_mapping: HashMap = HashMap::new(); - - // Process each script with new hash computation - for script in scripts { - // Create a duplicate of ScriptKind by matching the enum - let script_kind_for_hash = match script.kind { - ScriptKind::Script => ScriptKind::Script, - ScriptKind::Trigger => ScriptKind::Trigger, - ScriptKind::Failure => ScriptKind::Failure, - ScriptKind::Approval => ScriptKind::Approval, - ScriptKind::Preprocessor => ScriptKind::Preprocessor, - }; - let script_kind_for_db = match script.kind { - ScriptKind::Script => ScriptKind::Script, - ScriptKind::Trigger => ScriptKind::Trigger, - ScriptKind::Failure => ScriptKind::Failure, - ScriptKind::Approval => ScriptKind::Approval, - ScriptKind::Preprocessor => ScriptKind::Preprocessor, - }; - - // Create NewScript for hash computation - simplified approach - let new_script = NewScript { - path: script.path.clone(), - parent_hash: None, - summary: script.summary.clone(), - description: script.description.clone(), - content: script.content.clone(), - schema: None, // Keep it simple for hash computation - is_template: Some(script.is_template.unwrap_or(false)), - lock: script.lock.clone(), - language: script.language.clone(), - kind: Some(script_kind_for_hash), - tag: script.tag.clone(), - draft_only: script.draft_only, - envs: script.envs.clone(), - concurrent_limit: script.concurrent_limit, - concurrency_time_window_s: script.concurrency_time_window_s, - cache_ttl: script.cache_ttl, - dedicated_worker: script.dedicated_worker, - ws_error_handler_muted: Some(script.ws_error_handler_muted), - priority: script.priority, - timeout: script.timeout, - delete_after_use: script.delete_after_use, - restart_unless_cancelled: script.restart_unless_cancelled, - deployment_message: None, - concurrency_key: script.concurrency_key.clone(), - visible_to_runner_only: script.visible_to_runner_only, - no_main_func: script.no_main_func, - codebase: script.codebase.clone(), - has_preprocessor: script.has_preprocessor, - on_behalf_of_email: script.on_behalf_of_email.clone(), - assets: None, - }; - - // Generate new hash - let new_hash = hash_script(&new_script); - - // Store mapping for later reference updates - script_hash_mapping.insert(script.hash, new_hash); - - // Insert script with new hash - direct copy most fields - sqlx::query!( - r#"INSERT INTO script ( - workspace_id, hash, path, parent_hashes, summary, description, content, - created_by, created_at, archived, schema, deleted, is_template, - extra_perms, lock, lock_error_logs, language, kind, tag, draft_only, - envs, concurrent_limit, concurrency_time_window_s, cache_ttl, - dedicated_worker, ws_error_handler_muted, priority, timeout, - delete_after_use, restart_unless_cancelled, concurrency_key, - visible_to_runner_only, no_main_func, codebase, has_preprocessor, - on_behalf_of_email, assets - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, - $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, - $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, - $36, $37 - )"#, - target_workspace_id, - new_hash, - script.path, - script.parent_hashes.as_deref(), - script.summary, - script.description, - script.content, - target_username, - script.created_at, - script.archived, - script.schema, - script.deleted, - script.is_template, - script.extra_perms, - script.lock, - script.lock_error_logs, - script.language as _, - script_kind_for_db as _, - script.tag, - script.draft_only, - script.envs.as_deref(), - script.concurrent_limit, - script.concurrency_time_window_s, - script.cache_ttl, - script.dedicated_worker, - script.ws_error_handler_muted, - script.priority, - script.timeout, - script.delete_after_use, - script.restart_unless_cancelled, - script.concurrency_key, - script.visible_to_runner_only, - script.no_main_func, - script.codebase, - script.has_preprocessor, - script.on_behalf_of_email, - script.assets, - ) - .execute(&mut **tx) - .await?; - } - - Ok(script_hash_mapping) + Ok(()) } async fn clone_flows( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, target_workspace_id: &str, - target_username: &str, ) -> Result<()> { // First, clone flows without versions sqlx::query!( @@ -2846,15 +2780,14 @@ async fn clone_flows( ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, concurrency_key, versions, on_behalf_of_email, lock_error_logs ) - SELECT $2, path, summary, description, value, $3, edited_at, + SELECT $2, path, summary, description, value, edited_by, edited_at, archived, schema, extra_perms, NULL, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, concurrency_key, ARRAY[]::bigint[], on_behalf_of_email, lock_error_logs - FROM flow + FROM flow WHERE workspace_id = $1", source_workspace_id, target_workspace_id, - target_username, ) .execute(&mut **tx) .await?; @@ -2879,7 +2812,7 @@ async fn clone_flows( version.path, version.value, version.schema, - target_username, + version.created_by, version.created_at, ) .fetch_one(&mut **tx) @@ -2926,7 +2859,6 @@ async fn clone_apps( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, target_workspace_id: &str, - target_username: &str, ) -> Result> { // Get all apps from source workspace let apps = sqlx::query!( @@ -2961,30 +2893,32 @@ async fn clone_apps( app_id_mapping.insert(app.id, new_app_id); } - // Clone app versions - let app_versions = sqlx::query!( - "SELECT app_id, value, created_by, created_at, raw_app + { + // Clone app versions + let app_versions = sqlx::query!( + "SELECT app_id, value, created_by, created_at, raw_app FROM app_version WHERE app_id = ANY(SELECT id FROM app WHERE workspace_id = $1) ORDER BY app_id, created_at", - source_workspace_id - ) - .fetch_all(&mut **tx) - .await?; + source_workspace_id + ) + .fetch_all(&mut **tx) + .await?; - for version in app_versions { - if let Some(&new_app_id) = app_id_mapping.get(&version.app_id) { - sqlx::query!( - "INSERT INTO app_version (app_id, value, created_by, created_at, raw_app) + for version in app_versions { + if let Some(&new_app_id) = app_id_mapping.get(&version.app_id) { + sqlx::query!( + "INSERT INTO app_version (app_id, value, created_by, created_at, raw_app) VALUES ($1, $2, $3, $4, $5)", - new_app_id, - version.value, - target_username, - version.created_at, - version.raw_app, - ) - .execute(&mut **tx) - .await?; + new_app_id, + version.value, + version.created_by, + version.created_at, + version.raw_app, + ) + .execute(&mut **tx) + .await?; + } } } @@ -3023,7 +2957,7 @@ async fn clone_apps( sqlx::query!( "INSERT INTO app_script (app, hash, lock, code, code_sha256) - VALUES ($1, $2, $3, $4, $5)", + VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", new_app_id, new_hash, app_script.lock, @@ -3061,45 +2995,42 @@ async fn clone_workspace_dependencies( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, target_workspace_id: &str, - script_hash_mapping: &HashMap, ) -> Result<()> { - let dependencies = sqlx::query!( - "SELECT flow_path, runnable_path, script_hash, runnable_is_flow, app_path - FROM workspace_runnable_dependencies - WHERE workspace_id = $1", + // Clone workspace_runnable_dependencies + sqlx::query!( + "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id, app_path) + SELECT flow_path, runnable_path, script_hash, runnable_is_flow, $1, app_path + FROM workspace_runnable_dependencies + WHERE workspace_id = $2", + target_workspace_id, source_workspace_id ) - .fetch_all(&mut **tx) + .execute(&mut **tx) .await?; - for dep in dependencies { - let new_script_hash = if let Some(old_hash) = dep.script_hash { - script_hash_mapping.get(&old_hash).copied() - } else { - None - }; - - sqlx::query!( - "INSERT INTO workspace_runnable_dependencies ( - flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id, app_path - ) VALUES ($1, $2, $3, $4, $5, $6)", - dep.flow_path, - dep.runnable_path, - new_script_hash, - dep.runnable_is_flow, - target_workspace_id, - dep.app_path, - ) - .execute(&mut **tx) - .await?; - } + // Clone dependency_map to preserve import relationships + sqlx::query!( + "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) + SELECT $1, importer_path, importer_kind, imported_path, importer_node_id + FROM dependency_map + WHERE workspace_id = $2", + target_workspace_id, + source_workspace_id + ) + .execute(&mut **tx) + .await?; Ok(()) } +async fn deprecated_create_workspace_fork(_authed: ApiAuthed) -> Result { + return Err(Error::BadRequest("This API endpoint has been relocated. Your Windmill CLI version is outdated and needs to be updated.".to_string())); +} + async fn create_workspace_fork( authed: ApiAuthed, Extension(db): Extension, + Path(parent_workspace_id): Path, Json(nw): Json, ) -> Result { // if *CREATE_WORKSPACE_REQUIRE_SUPERADMIN { @@ -3124,29 +3055,6 @@ async fn create_workspace_fork( let forked_id = nw.id; - // Determine username early so we can use it in workspace creation - let automate_username_creation = sqlx::query_scalar!( - "SELECT value FROM global_settings WHERE name = $1", - AUTOMATE_USERNAME_CREATION_SETTING, - ) - .fetch_optional(&mut *tx) - .await? - .map(|v| v.as_bool()) - .flatten() - .unwrap_or(false); - - let username = if automate_username_creation { - if nw.username.is_some() && nw.username.unwrap().len() > 0 { - return Err(Error::BadRequest( - "username is not allowed when username creation is automated".to_string(), - )); - } - get_instance_username_or_create_pending(&mut tx, &authed.email).await? - } else { - nw.username - .ok_or(Error::BadRequest("username is required".to_string()))? - }; - sqlx::query!( "INSERT INTO workspace (id, name, owner, parent_workspace_id) @@ -3154,7 +3062,7 @@ async fn create_workspace_fork( forked_id, nw.name, authed.email, - nw.parent_workspace_id, + parent_workspace_id, ) .execute(&mut *tx) .await?; @@ -3168,31 +3076,22 @@ async fn create_workspace_fork( ) .execute(&mut *tx) .await?; - let key = rd_string(64); - sqlx::query!( - "INSERT INTO workspace_key - (workspace_id, kind, key) - VALUES ($1, 'cloud', $2)", - forked_id, - &key - ) - .execute(&mut *tx) - .await?; sqlx::query!( "INSERT INTO usr - (workspace_id, email, username, is_admin) - VALUES ($1, $2, $3, $4)", + (workspace_id, email, username, is_admin) + SELECT $1, email, username, is_admin FROM usr + WHERE workspace_id = $3 AND email = $2 + ", forked_id, authed.email, - username, - authed.is_admin, + parent_workspace_id, ) .execute(&mut *tx) .await?; // Clone all data from the parent workspace using Rust implementation - clone_workspace_data(&mut tx, &nw.parent_workspace_id, &forked_id, &username, &db).await?; + clone_workspace_data(&mut tx, &parent_workspace_id, &forked_id).await?; sqlx::query!( "INSERT INTO workspace_invite (workspace_id, email, is_admin, operator) @@ -3200,7 +3099,7 @@ async fn create_workspace_fork( FROM usr WHERE workspace_id = $2", &forked_id, - &nw.parent_workspace_id + &parent_workspace_id ) .execute(&mut *tx) .await?; @@ -3366,7 +3265,7 @@ async fn invite_user( "INSERT INTO workspace_invite (workspace_id, email, is_admin, operator) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, email) - DO UPDATE SET is_admin = $3, operator = $4", + DO UPDATE SET is_admin = EXCLUDED.is_admin, operator = EXCLUDED.operator", &w_id, nu.email, nu.is_admin, @@ -3600,6 +3499,42 @@ async fn get_workspace_name( Ok(workspace) } +async fn get_dependency_map( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, +) -> JsonResult> { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = user_db.begin(&authed).await?; + let dmap = sqlx::query_as!( + DependencyMap, + " + SELECT workspace_id, importer_path, importer_kind::text, imported_path, importer_node_id + FROM dependency_map WHERE workspace_id = $1", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(Json(dmap)) +} + +#[axum::debug_handler] +async fn rebuild_dependency_map( + Extension(db): Extension, + Path(w_id): Path, + authed: ApiAuthed, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + if *CLOUD_HOSTED { + return Err(Error::BadRequest("Disabled on Cloud".into())); + } + ScopedDependencyMap::rebuild_map(&w_id, &db).await +} + #[derive(Deserialize)] struct ChangeWorkspaceName { new_name: String, diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index fb7afaccaa..74732c9f22 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -102,6 +102,10 @@ struct ScriptMetadata { pub has_preprocessor: Option, #[serde(skip_serializing_if = "Option::is_none")] pub on_behalf_of_email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub debounce_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub debounce_delay_s: Option, } pub fn is_none_or_false(val: &Option) -> bool { @@ -204,7 +208,6 @@ where "edited_by", "archived", "has_draft", - "draft_only", "error", "last_server_ping", "server_id", @@ -354,6 +357,7 @@ pub(crate) async fn tarball_workspace( { let scripts = sqlx::query_as::<_, Script>( "SELECT * FROM script as o WHERE workspace_id = $1 AND archived = false + AND (draft_only IS NULL OR draft_only = false) AND created_at = (select max(created_at) from script where path = o.path AND \ workspace_id = $1)", ) @@ -426,6 +430,8 @@ pub(crate) async fn tarball_workspace( concurrency_key: script.concurrency_key, has_preprocessor: script.has_preprocessor, on_behalf_of_email: script.on_behalf_of_email, + debounce_key: script.debounce_key, + debounce_delay_s: script.debounce_delay_s, }; let metadata_str = serde_json::to_string_pretty(&metadata).unwrap(); archive @@ -476,7 +482,7 @@ pub(crate) async fn tarball_workspace( "SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.draft_only, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of_email, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] - WHERE flow.workspace_id = $1 AND flow.archived = false", + WHERE flow.workspace_id = $1 AND flow.archived = false AND (flow.draft_only IS NULL OR flow.draft_only = false)", ) .bind(&w_id) .fetch_all(&mut *tx) @@ -508,7 +514,9 @@ pub(crate) async fn tarball_workspace( && var.value.is_some() && var.is_secret { - var.value = Some(decrypt(&mc, var.value.unwrap())?); + var.value = Some(decrypt(&mc, var.value.unwrap()).map_err(|e| { + Error::internal_err(format!("Error decrypting variable {}: {}", var.path, e)) + })?); } let var_str = &to_string_without_metadata(&var, false, None).unwrap(); archive @@ -522,7 +530,8 @@ pub(crate) async fn tarball_workspace( "SELECT app.id, app.path, app.summary, app.versions, app.policy, app.custom_path, app.extra_perms, app_version.value, app_version.created_at, app_version.created_by from app, app_version - WHERE app.workspace_id = $1 AND app_version.id = app.versions[array_upper(app.versions, 1)] AND app_version.raw_app IS false", + WHERE app.workspace_id = $1 AND app_version.id = app.versions[array_upper(app.versions, 1)] AND app_version.raw_app IS false + AND (app.draft_only IS NULL OR app.draft_only = false)", ) .bind(&w_id) .fetch_all(&mut *tx) diff --git a/backend/windmill-api/src/workspaces_extra.rs b/backend/windmill-api/src/workspaces_extra.rs index 02996c0d32..2df23f3ce9 100644 --- a/backend/windmill-api/src/workspaces_extra.rs +++ b/backend/windmill-api/src/workspaces_extra.rs @@ -3,6 +3,7 @@ use crate::db::ApiAuthed; use crate::workspaces::{check_w_id_conflict, CREATE_WORKSPACE_REQUIRE_SUPERADMIN, WM_FORK_PREFIX}; use crate::{db::DB, utils::require_super_admin}; +use axum::extract::Query; use axum::{ extract::{Extension, Path}, Json, @@ -415,10 +416,16 @@ pub(crate) async fn change_workspace_id( )) } +#[derive(Deserialize)] +pub(crate) struct DeleteWorkspaceQuery { + pub(crate) only_delete_forks: Option, +} + pub(crate) async fn delete_workspace( Extension(db): Extension, Path(w_id): Path, authed: ApiAuthed, + Query(dwq): Query, ) -> Result { let w_id = match w_id.as_str() { "starter" => Err(Error::BadRequest( @@ -429,11 +436,22 @@ pub(crate) async fn delete_workspace( )), _ => Ok(w_id), }?; + + if dwq.only_delete_forks.unwrap_or(false) && !w_id.starts_with(WM_FORK_PREFIX) { + return Err(Error::BadRequest( + "Cannot delete this workspace because it is not a workspace fork.".to_string(), + )); + } + let mut tx = db.begin().await?; if !(w_id.starts_with(WM_FORK_PREFIX) && is_workspace_owner(&authed, &w_id, &mut tx).await?) { require_super_admin(&db, &authed.email).await?; } + sqlx::query!("DELETE FROM workspace_env WHERE workspace_id = $1", &w_id) + .execute(&mut *tx) + .await?; + sqlx::query!("DELETE FROM dependency_map WHERE workspace_id = $1", &w_id) .execute(&mut *tx) .await?; diff --git a/backend/windmill-audit/Cargo.toml b/backend/windmill-audit/Cargo.toml index 5a9ea376b6..6e4fc3fd12 100644 --- a/backend/windmill-audit/Cargo.toml +++ b/backend/windmill-audit/Cargo.toml @@ -20,3 +20,4 @@ chrono.workspace = true serde_json.workspace = true tracing.workspace = true windmill-common = { workspace = true, default-features = false } +lazy_static.workspace = true \ No newline at end of file diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index ab17eb81cf..8317940079 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -65,6 +65,7 @@ prometheus = { workspace = true, optional = true } aws-config = { workspace = true, optional = true } aws-sdk-sts = { workspace = true, optional = true } base64.workspace = true +bitflags.workspace = true aws-smithy-types-convert = { workspace = true, optional = true } indexmap.workspace = true @@ -95,6 +96,8 @@ futures.workspace = true tempfile.workspace = true systemstat.workspace = true size.workspace = true +globset.workspace = true +rmcp = { version = "0.8.1", features = ["client", "transport-streamable-http-client", "transport-streamable-http-client-reqwest"] } opentelemetry-semantic-conventions = { workspace = true, optional = true } opentelemetry-otlp = { workspace = true, optional = true } diff --git a/backend/windmill-common/src/agent_workers.rs b/backend/windmill-common/src/agent_workers.rs index 74626522e6..322ca1c4fb 100644 --- a/backend/windmill-common/src/agent_workers.rs +++ b/backend/windmill-common/src/agent_workers.rs @@ -11,7 +11,7 @@ use std::time::Duration; use reqwest_middleware::ClientBuilder; use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; -use crate::{jwt::decode_without_verify, worker::HttpClient}; +use crate::{jwt::decode_without_verify, utils::configure_client, worker::HttpClient}; lazy_static! { pub static ref AGENT_TOKEN: String = std::env::var("AGENT_TOKEN").unwrap_or_default(); @@ -35,13 +35,17 @@ pub struct AgentAuth { pub const AGENT_JWT_PREFIX: &str = "jwt_agent_"; -pub fn build_agent_http_client(worker_suffix: &str) -> HttpClient { +pub fn build_agent_http_client( + worker_suffix: &str, + agent_token: Option, + base_internal_url: Option, +) -> HttpClient { let client = ClientBuilder::new( - reqwest::Client::builder() + configure_client(reqwest::Client::builder() .pool_max_idle_per_host(10) .pool_idle_timeout(Duration::from_secs(60)) .connect_timeout(Duration::from_secs(10)) - .timeout(Duration::from_secs(30)) + .timeout(Duration::from_secs(30))) .default_headers({ let mut headers = reqwest::header::HeaderMap::new(); headers.insert( @@ -52,7 +56,9 @@ pub fn build_agent_http_client(worker_suffix: &str) -> HttpClient { "{}{}_{}", AGENT_JWT_PREFIX, worker_suffix, - AGENT_TOKEN.trim_start_matches(AGENT_JWT_PREFIX), + agent_token + .unwrap_or(AGENT_TOKEN.clone()) + .trim_start_matches(AGENT_JWT_PREFIX) ); headers.insert( "Authorization", @@ -67,7 +73,8 @@ pub fn build_agent_http_client(worker_suffix: &str) -> HttpClient { ExponentialBackoff::builder().build_with_max_retries(5), )) .build(); - HttpClient(client) + + HttpClient { client, base_internal_url } } #[derive(Deserialize, Serialize)] diff --git a/backend/windmill-common/src/ai_providers.rs b/backend/windmill-common/src/ai_providers.rs index 577f613f21..46d5d687c3 100644 --- a/backend/windmill-common/src/ai_providers.rs +++ b/backend/windmill-common/src/ai_providers.rs @@ -10,6 +10,9 @@ lazy_static::lazy_static! { static ref OPENAI_AZURE_BASE_PATH: Option = std::env::var("OPENAI_AZURE_BASE_PATH").ok(); } +pub const AZURE_API_VERSION: &str = "2025-04-01-preview"; +pub const OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; + #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] #[serde(rename_all = "lowercase")] pub enum AIProvider { @@ -78,6 +81,39 @@ impl AIProvider { pub fn is_anthropic(&self) -> bool { matches!(self, AIProvider::Anthropic) } + + /// Check if this provider/URL combination represents Azure OpenAI + pub fn is_azure_openai(&self, base_url: &str) -> bool { + (matches!(self, AIProvider::OpenAI) && base_url != OPENAI_BASE_URL) + || matches!(self, AIProvider::AzureOpenAI) + } + + /// Build Azure OpenAI URL with deployment model path + pub fn build_azure_openai_url(base_url: &str, model: &str, path: &str) -> String { + let base_url = base_url.trim_end_matches('/'); + + if base_url.ends_with("/deployments") { + format!("{}/{}/{}", base_url, model, path) + } else if base_url.ends_with("/openai") { + format!("{}/deployments/{}/{}", base_url, model, path) + } else { + format!("{}/{}", base_url, path) + } + } + + /// Extract model from request body (needed for Azure deployments) + pub fn extract_model_from_body(body: &[u8]) -> Result { + #[derive(serde::Deserialize)] + struct ModelRequest { + model: String, + } + + let model_request: ModelRequest = serde_json::from_slice(body).map_err(|e| { + Error::internal_err(format!("Failed to parse request body for model: {}", e)) + })?; + + Ok(model_request.model) + } } impl TryFrom<&str> for AIProvider { diff --git a/backend/windmill-common/src/apps.rs b/backend/windmill-common/src/apps.rs index b4552111e0..e27504a9f6 100644 --- a/backend/windmill-common/src/apps.rs +++ b/backend/windmill-common/src/apps.rs @@ -9,8 +9,11 @@ use std::{collections::HashMap, sync::Arc}; use serde::{Deserialize, Serialize}; +use serde_json::{from_value, Value}; use tokio::sync::RwLock; +use crate::{error, scripts::ScriptLang}; + lazy_static::lazy_static! { pub static ref APP_WORKSPACED_ROUTE: Arc> = Arc::new(RwLock::new(false)); } @@ -33,3 +36,70 @@ pub struct ListAppQuery { pub struct RawAppValue { pub files: HashMap, } + +pub struct AppInlineScript { + pub language: Option, + pub content: String, + pub lock: Option, +} + +/// Traverse FlowValue while invoking provided by caller callback on leafs +// #[async_recursion::async_recursion(?Send)] +pub fn traverse_app_inline_scripts< + C: FnMut(AppInlineScript, Option) -> error::Result<()>, +>( + value: &Value, + // Set to None. + container_id: Option, + cb: &mut C, +) -> error::Result<()> { + match value { + Value::Object(object) => { + if let Some(Value::Object(script)) = object.get("inlineScript") { + let (language, lock, code) = ( + script + .get("language") + .cloned() + .map(|v| from_value::(v).ok()) + .flatten(), + script + .get("lock") + .and_then(Value::as_str) + .map(str::to_owned), + script + .get("content") + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or(error::Error::internal_err( + "Missing `content` in inlineScript".to_string(), + ))?, + ); + if language.is_some() { + cb( + AppInlineScript { language, content: code.to_owned(), lock }, + container_id.clone(), + )?; + } + } else { + for (_, value) in object { + traverse_app_inline_scripts( + value, + object + .get("id") + .and_then(Value::as_str) + .map(str::to_owned) + .or(container_id.clone()), + cb, + )?; + } + } + } + Value::Array(array) => { + for value in array { + traverse_app_inline_scripts(value, container_id.clone(), cb)?; + } + } + _ => {} + } + Ok(()) +} diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index c2b5b5dffd..e48cf3728e 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -140,13 +140,26 @@ pub struct JWTAuthClaims { pub groups: Vec, pub folders: Vec<(String, bool, bool)>, pub label: Option, - pub workspace_id: String, + pub workspace_id: Option, + pub workspace_ids: Option>, pub exp: usize, pub job_id: Option, pub scopes: Option>, pub audit_span: Option, } +impl JWTAuthClaims { + pub fn allowed_in_workspace(&self, w_id: &str) -> bool { + self.workspace_id + .as_ref() + .is_some_and(|token_w_id| w_id == token_w_id) + || self + .workspace_ids + .as_ref() + .is_some_and(|token_w_ids| token_w_ids.iter().any(|token_w_id| w_id == token_w_id)) + } +} + #[derive(Deserialize, Debug)] pub struct JobPerms { pub email: String, @@ -411,7 +424,8 @@ pub async fn create_jwt_token( groups: authed.groups.clone(), folders: authed.folders.clone(), label, - workspace_id: workspace_id.to_string(), + workspace_id: Some(workspace_id.to_string()), + workspace_ids: None, exp: (chrono::Utc::now() + chrono::Duration::seconds(expires_in_seconds as i64)).timestamp() as usize, job_id: job_id.map(|id| id.to_string()), diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index 4890b9715b..3d1524916a 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -15,6 +15,7 @@ use crate::{ scripts::{ScriptHash, ScriptLang}, }; use anyhow::anyhow; +use serde_json::value::to_raw_value; #[cfg(feature = "scoped_cache")] use std::thread::ThreadId; @@ -284,10 +285,30 @@ pub struct FlowData { pub flow: FlowValue, } +/// !!!Shouldn't be used. Reverted optimization for ai agent steps.!!! +#[derive(Deserialize)] +struct RevertedFlowNodeFlow { + value: FlowValue, +} + impl FlowData { pub fn from_raw(raw_flow: Box) -> error::Result { - let flow = serde_json::from_str(raw_flow.get())?; - Ok(Self { raw_flow, flow }) + match serde_json::from_str::(raw_flow.get()) { + Ok(flow) => Ok(FlowData { raw_flow, flow }), + _ => { + // fallback for compatibility with bad version 1.560.0 + // TODO: remove this in a future version. Reverted optimization for ai agent steps. + let flow_node_flow = serde_json::from_str::(raw_flow.get()) + .map_err(|e| { + error::Error::internal_err(format!( + "Failed to parse as RevertedFlowNodeFlow: {}", + e + )) + })?; + let raw_flow = to_raw_value(&flow_node_flow.value)?; + Ok(FlowData { raw_flow, flow: flow_node_flow.value }) + } + } } pub fn value(&self) -> &FlowValue { @@ -837,10 +858,12 @@ pub mod job { match (kind, hash.map(|ScriptHash(id)| id)) { (FlowDependencies, Some(id)) => flow::fetch_version(db, id).await, (FlowNode, Some(id)) => flow::fetch_flow(db, FlowNodeId(id)).await, - (Flow, Some(id)) => match flow::fetch_version_lite(db, id).await { - Ok(raw_flow) => Ok(raw_flow), - Err(_) => flow::fetch_version(db, id).await, - }, + (Flow, Some(id)) | (SingleStepFlow, Some(id)) => { + match flow::fetch_version_lite(db, id).await { + Ok(raw_flow) => Ok(raw_flow), + Err(_) => flow::fetch_version(db, id).await, + } + } _ => Err(error::Error::internal_err(format!( "Isn't a flow job {:?}", kind diff --git a/backend/windmill-common/src/client.rs b/backend/windmill-common/src/client.rs index 9507100eb5..b05fdfa2f7 100644 --- a/backend/windmill-common/src/client.rs +++ b/backend/windmill-common/src/client.rs @@ -39,8 +39,8 @@ impl AuthedClient { .send() .await .map_err(|e| { - tracing::error!("Error executing get request from authed http client to {url} with query {query:?}: {e}"); - anyhow::anyhow!("Error executing get request from authed http client to {url} with query {query:?}: {e}") + tracing::error!("Error executing get request from authed http client to {url} with query {query:?}: {e:#?}"); + anyhow::anyhow!("Error executing get request from authed http client to {url} with query {query:?}: {e:#?}") }) } @@ -206,4 +206,42 @@ impl AuthedClient { _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?, } } + + pub async fn download_s3_file( + &self, + workspace_id: &str, + file_key: &str, + storage: Option, + ) -> anyhow::Result { + let mut query = vec![("file_key", file_key.to_string())]; + if let Some(storage) = storage { + query.push(("storage", storage)); + } + let response = self + .force_client + .as_ref() + .unwrap_or(&HTTP_CLIENT) + .get(&format!( + "{}/api/w/{}/job_helpers/download_s3_file", + self.base_internal_url, workspace_id + )) + .query(&query) + .header( + reqwest::header::AUTHORIZATION, + reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token)) + .map_err(|e| anyhow::anyhow!(e.to_string()))?, + ) + .send() + .await + .context("Failed to send download_s3_file request") + .map_err(|e| anyhow::anyhow!(e.to_string()))?; + + match response.status().as_u16() { + 200u16 => Ok(response + .bytes() + .await + .context("Failed to read response bytes")?), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), + } + } } diff --git a/backend/windmill-common/src/db.rs b/backend/windmill-common/src/db.rs index c25b0f5c6a..1aa5e57ea0 100644 --- a/backend/windmill-common/src/db.rs +++ b/backend/windmill-common/src/db.rs @@ -2,7 +2,7 @@ use sqlx::{Acquire, Pool, Postgres, Transaction}; pub type DB = Pool; -#[derive(Clone, Debug, Hash)] +#[derive(Clone, Debug, Hash, Eq, PartialEq)] pub struct Authed { pub email: String, pub username: String, diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index 1c41bdc9b5..8048b59586 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -84,6 +84,8 @@ pub enum Error { ArgumentErr(String), #[error("{1}")] Generic(StatusCode, String), + #[error("{feature} is unavailable due to some workers being behind. Do not use the feature or make sure all workers run at least {min_version}")] + WorkersAreBehind { feature: String, min_version: String }, } impl Error { @@ -188,6 +190,12 @@ impl From for Error { } } +impl From for Error { + fn from(value: tokio::time::error::Elapsed) -> Self { + Self::InternalErr(value.to_string()) + } +} + impl Error { /// https://docs.rs/anyhow/1/anyhow/struct.Error.html#display-representations pub fn alt(&self) -> String { diff --git a/backend/windmill-common/src/external_ip.rs b/backend/windmill-common/src/external_ip.rs index 0831bb54bc..d0a8507b44 100644 --- a/backend/windmill-common/src/external_ip.rs +++ b/backend/windmill-common/src/external_ip.rs @@ -12,6 +12,7 @@ //! connections to be from whitelisted IP addresses. use std::time::Duration; +use crate::utils::configure_client; pub async fn get_ip() -> anyhow::Result { tokio::select! { @@ -19,9 +20,9 @@ pub async fn get_ip() -> anyhow::Result { _ = tokio::time::sleep(Duration::from_secs(10)) => { return Err(anyhow::anyhow!("Expected to get ip under 10s")) }, - ip = reqwest::ClientBuilder::new() + ip = configure_client(reqwest::ClientBuilder::new() .connect_timeout(Duration::from_secs(5)) - .timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(5))) .build()? .get("https://hub.windmill.dev/getip") .send() => Ok(ip? diff --git a/backend/windmill-common/src/flow_conversations.rs b/backend/windmill-common/src/flow_conversations.rs new file mode 100644 index 0000000000..8826f85b94 --- /dev/null +++ b/backend/windmill-common/src/flow_conversations.rs @@ -0,0 +1,50 @@ +use serde::{Deserialize, Serialize}; +use sqlx; +use uuid::Uuid; + +use crate::error::Result; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, sqlx::Type)] +#[sqlx(type_name = "MESSAGE_TYPE", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum MessageType { + User, + Assistant, + System, + Tool, +} + +/// Add a message to a conversation using an existing transaction +pub async fn add_message_to_conversation_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + conversation_id: Uuid, + job_id: Option, + content: &str, + message_type: MessageType, + step_name: Option<&str>, + success: bool, +) -> Result<()> { + // Insert the message + sqlx::query!( + "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, step_name, success) + VALUES ($1, $2, $3, $4, $5, $6)", + conversation_id, + message_type as MessageType, + content, + job_id, + step_name, + success + ) + .execute(&mut **tx) + .await?; + + // Update conversation updated_at timestamp + sqlx::query!( + "UPDATE flow_conversation SET updated_at = NOW() WHERE id = $1", + conversation_id + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} diff --git a/backend/windmill-common/src/flow_status.rs b/backend/windmill-common/src/flow_status.rs index 1ae95843b7..44da5f6639 100644 --- a/backend/windmill-common/src/flow_status.rs +++ b/backend/windmill-common/src/flow_status.rs @@ -43,6 +43,12 @@ pub struct FlowStatus { pub approval_conditions: Option, #[serde(skip_serializing_if = "Option::is_none")] pub restarted_from: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_job: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_input_enabled: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub memory_id: Option, } #[derive(Serialize, Deserialize, Debug, Clone, Default)] @@ -111,6 +117,50 @@ pub struct FlowCleanupModule { pub flow_jobs_to_clean: Vec, } +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct FlowJobsDuration { + pub started_at: Vec>>, + pub duration_ms: Vec>, +} + +impl FlowJobsDuration { + pub fn set(&mut self, position: Option, value: &Option) { + if let Some(position) = position { + if position >= self.started_at.len() + || position >= self.duration_ms.len() + || value.is_none() + { + return; + } + let value = value.clone().unwrap(); + self.started_at[position] = Some(value.started_at); + self.duration_ms[position] = Some(value.duration_ms); + } + } + + pub fn push(&mut self, value: &Option) { + self.started_at.push(value.as_ref().map(|x| x.started_at)); + self.duration_ms.push(value.as_ref().map(|x| x.duration_ms)); + } + + pub fn new(n: usize) -> Self { + Self { started_at: vec![None; n], duration_ms: vec![None; n] } + } +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct FlowJobDuration { + pub started_at: chrono::DateTime, + pub duration_ms: i64, +} + +impl FlowJobsDuration { + pub fn truncate(&mut self, n: usize) { + self.started_at.truncate(n); + self.duration_ms.truncate(n); + } +} + #[derive(Deserialize)] struct UntaggedFlowStatusModule { #[serde(rename = "type")] @@ -122,6 +172,7 @@ struct UntaggedFlowStatusModule { iterator: Option, flow_jobs: Option>, flow_jobs_success: Option>>, + flow_jobs_duration: Option, branch_chosen: Option, branchall: Option, parallel: Option, @@ -136,7 +187,18 @@ struct UntaggedFlowStatusModule { #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(tag = "type", rename_all = "snake_case")] pub enum AgentAction { - ToolCall { job_id: uuid::Uuid, function_name: String, module_id: String }, + ToolCall { + job_id: uuid::Uuid, + function_name: String, + module_id: String, + }, + McpToolCall { + call_id: uuid::Uuid, + function_name: String, + resource_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + arguments: Option, + }, Message {}, } @@ -167,6 +229,8 @@ pub enum FlowStatusModule { #[serde(skip_serializing_if = "Option::is_none")] flow_jobs_success: Option>>, #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs_duration: Option, + #[serde(skip_serializing_if = "Option::is_none")] branch_chosen: Option, #[serde(skip_serializing_if = "Option::is_none")] branchall: Option, @@ -187,6 +251,8 @@ pub enum FlowStatusModule { #[serde(skip_serializing_if = "Option::is_none")] flow_jobs_success: Option>>, #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs_duration: Option, + #[serde(skip_serializing_if = "Option::is_none")] branch_chosen: Option, #[serde(default)] #[serde(skip_serializing_if = "Vec::is_empty")] @@ -207,6 +273,8 @@ pub enum FlowStatusModule { #[serde(skip_serializing_if = "Option::is_none")] flow_jobs_success: Option>>, #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs_duration: Option, + #[serde(skip_serializing_if = "Option::is_none")] branch_chosen: Option, #[serde(skip_serializing_if = "Vec::is_empty")] failed_retries: Vec, @@ -260,6 +328,7 @@ impl<'de> Deserialize<'de> for FlowStatusModule { iterator: untagged.iterator, flow_jobs: untagged.flow_jobs, flow_jobs_success: untagged.flow_jobs_success, + flow_jobs_duration: untagged.flow_jobs_duration, branch_chosen: untagged.branch_chosen, branchall: untagged.branchall, parallel: untagged.parallel.unwrap_or(false), @@ -277,6 +346,7 @@ impl<'de> Deserialize<'de> for FlowStatusModule { .ok_or_else(|| serde::de::Error::missing_field("job"))?, flow_jobs: untagged.flow_jobs, flow_jobs_success: untagged.flow_jobs_success, + flow_jobs_duration: untagged.flow_jobs_duration, branch_chosen: untagged.branch_chosen, approvers: untagged.approvers.unwrap_or_default(), failed_retries: untagged.failed_retries.unwrap_or_default(), @@ -293,6 +363,7 @@ impl<'de> Deserialize<'de> for FlowStatusModule { .ok_or_else(|| serde::de::Error::missing_field("job"))?, flow_jobs: untagged.flow_jobs, flow_jobs_success: untagged.flow_jobs_success, + flow_jobs_duration: untagged.flow_jobs_duration, branch_chosen: untagged.branch_chosen, failed_retries: untagged.failed_retries.unwrap_or_default(), agent_actions: untagged.agent_actions, @@ -358,6 +429,15 @@ impl FlowStatusModule { } } + pub fn flow_jobs_duration(&self) -> Option { + match self { + FlowStatusModule::InProgress { flow_jobs_duration, .. } => flow_jobs_duration.clone(), + FlowStatusModule::Success { flow_jobs_duration, .. } => flow_jobs_duration.clone(), + FlowStatusModule::Failure { flow_jobs_duration, .. } => flow_jobs_duration.clone(), + _ => None, + } + } + pub fn job_result(&self) -> Option { self.flow_jobs() .map(JobResult::ListJob) @@ -442,6 +522,9 @@ impl FlowStatus { retry: RetryStatus { fail_count: 0, failed_jobs: vec![] }, restarted_from: None, user_states: HashMap::new(), + stream_job: None, + chat_input_enabled: f.chat_input_enabled, + memory_id: None, } } diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 962ffd6c11..fcfac158d3 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -23,7 +23,7 @@ use crate::{ assets::AssetWithAltAccessType, cache, db::DB, - error::Error, + error::{Error, Result as WindmillResult}, more_serde::{default_empty_string, default_id, default_null, default_true, is_default}, scripts::{Schema, ScriptHash, ScriptLang}, worker::{to_raw_value, Connection}, @@ -93,12 +93,51 @@ pub struct ListableFlow { pub deployment_msg: Option, } -#[derive(Debug, Deserialize, sqlx::FromRow)] +fn validate_retry(retry: &Retry, module_id: &str) -> WindmillResult<()> { + if retry.exponential.attempts > 0 && retry.exponential.seconds == 0 { + return Err(Error::BadRequest(format!( + "Module '{}': Exponential backoff base (seconds) must be greater than 0. A base of 0 would cause immediate retries.", + module_id + ))); + } + Ok(()) +} + +fn validate_flow_value<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let raw_value = Box::::deserialize(deserializer)?; + + let flow_value: FlowValue = serde_json::from_str(raw_value.get()) + .map_err(|e| serde::de::Error::custom(format!("Invalid flow value: {}", e)))?; + + FlowModule::traverse_modules(&flow_value.modules, &mut |module| { + if let Some(ref retry) = module.retry { + validate_retry(retry, &module.id)?; + } + return Ok(()); + }) + .map_err(|e| serde::de::Error::custom(e.to_string()))?; + + if let Some(ref _failure_module) = flow_value.failure_module { + //add validation logic here for failure module + } + + if let Some(ref _preprocessor_module) = flow_value.preprocessor_module { + //add validation logic here for preprocessor module + } + + Ok(raw_value) +} + +#[derive(Debug, Serialize, Deserialize, sqlx::FromRow)] pub struct NewFlow { pub path: String, pub summary: String, pub description: Option, - pub value: serde_json::Value, + #[serde(deserialize_with = "validate_flow_value")] + pub value: Box, pub schema: Option, pub draft_only: Option, pub tag: Option, @@ -107,6 +146,15 @@ pub struct NewFlow { pub deployment_message: Option, pub visible_to_runner_only: Option, pub on_behalf_of_email: Option, + pub ws_error_handler_muted: Option, +} + +impl NewFlow { + pub fn parse_flow_value(&self) -> crate::error::Result { + serde_json::from_str(self.value.get()).map_err(|e| { + crate::error::Error::InternalErr(format!("Failed to parse flow value: {}", e)) + }) + } } #[derive(Deserialize, Serialize, Debug, Clone, Default)] @@ -122,10 +170,17 @@ pub struct FlowValue { #[serde(skip_serializing_if = "is_default")] pub same_worker: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_key: Option, #[serde(skip_serializing_if = "Option::is_none")] pub concurrent_limit: Option, #[serde(skip_serializing_if = "Option::is_none")] pub concurrency_time_window_s: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub debounce_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub debounce_delay_s: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub skip_expr: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -136,7 +191,7 @@ pub struct FlowValue { // Priority at the flow level pub priority: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency_key: Option, + pub chat_input_enabled: Option, } impl FlowValue { @@ -146,10 +201,10 @@ impl FlowValue { .preprocessor_module .as_deref() .with_context(|| format!("no preprocessor module")), - Step::Step(i) => self + Step::Step { idx, .. } => self .modules - .get(i) - .with_context(|| format!("no module found at index: {i}")), + .get(idx) + .with_context(|| format!("no module found at index: {idx}")), Step::FailureStep => self .failure_module .as_deref() @@ -158,11 +213,49 @@ impl FlowValue { flow_module } + + /// Traverse FlowValue while invoking provided by caller callback on leafs + // #[async_recursion::async_recursion(?Send)] + // TODO: We may be want this async. + pub fn traverse_leafs crate::error::Result<()>>( + modules: Vec<&FlowModule>, + cb: &mut C, + ) -> crate::error::Result<()> { + use FlowModuleValue::*; + for module in modules { + match serde_json::from_str::(module.value.get())? { + s @ (Script { .. } + | RawScript { .. } + | Flow { .. } + | FlowScript { .. } + | Identity) => cb(&s, &module.id)?, + ForloopFlow { modules, .. } | WhileloopFlow { modules, .. } => { + Self::traverse_leafs(modules.iter().collect(), cb)? + } + AIAgent { tools, .. } => { + for tool in tools { + match &tool.value { + ToolValue::FlowModule(module_value) => cb(module_value, &tool.id)?, + ToolValue::Mcp(_) => { + // MCP tools don't have a FlowModuleValue to traverse + } + } + } + } + BranchOne { branches, .. } | BranchAll { branches, .. } => { + for branch in branches { + Self::traverse_leafs(branch.modules.iter().collect(), cb)?; + } + } + } + } + Ok(()) + } } #[derive(Debug, Copy, Clone)] pub enum Step { - Step(usize), + Step { idx: usize, len: usize }, PreprocessorStep, FailureStep, } @@ -172,7 +265,7 @@ impl Step { if step < 0 { Step::PreprocessorStep } else if (step as usize) < len { - Step::Step(step as usize) + Step::Step { idx: step as usize, len } } else { Step::FailureStep } @@ -180,13 +273,13 @@ impl Step { pub fn get_step_index(&self) -> Option { match self { - Step::Step(index) => Some(*index), + Step::Step { idx, .. } => Some(*idx), _ => None, } } pub fn is_index_step(&self) -> bool { - matches!(self, Step::Step(_)) + matches!(self, Step::Step { .. }) } pub fn is_preprocessor_step(&self) -> bool { @@ -196,6 +289,10 @@ impl Step { pub fn is_failure_step(&self) -> bool { matches!(self, Step::FailureStep) } + + pub fn is_last_step(&self) -> bool { + matches!(self, Step::Step { idx, len } if *idx == len - 1) + } } #[derive(Default, Deserialize, Serialize, Debug, Clone)] @@ -359,6 +456,8 @@ pub struct FlowModule { pub skip_if: Option, #[serde(skip_serializing_if = "Option::is_none")] pub apply_preprocessor: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub pass_flow_input_directly: Option, } #[derive(Deserialize, Serialize, Debug, Clone)] @@ -371,14 +470,24 @@ pub struct FlowModuleValueWithParallel { #[serde(rename = "type")] pub type_: String, pub parallel: Option, - pub parallelism: Option, + #[serde( + default, + deserialize_with = "raw_value_to_input_transform::<_, u16>", + skip_serializing_if = "Option::is_none" + )] + pub parallelism: Option, } #[derive(Deserialize)] pub struct FlowModuleValueWithSkipFailures { pub skip_failures: Option, pub parallel: Option, - pub parallelism: Option, + #[serde( + default, + deserialize_with = "raw_value_to_input_transform::<_, u16>", + skip_serializing_if = "Option::is_none" + )] + pub parallelism: Option, } #[derive(Deserialize)] @@ -418,6 +527,10 @@ impl FlowModule { .map_err(crate::error::to_anyhow) } + pub fn is_ai_agent(&self) -> bool { + self.get_type().is_ok_and(|x| x == "aiagent") + } + pub fn is_simple(&self) -> bool { //todo: flow modules could also be simple execpt for the fact that the case of having single parallel flow approval step is not handled well (Create SuspendedTimeout) self.get_type() @@ -434,6 +547,64 @@ impl FlowModule { .map_err(crate::error::to_anyhow) .map(|x| x.r#type) } + + pub fn traverse_modules crate::error::Result<()>>( + modules: &Vec, + cb: &mut C, + ) -> crate::error::Result<()> { + for module in modules { + cb(module)?; + match module + .get_value() + .map_err(|e| Error::BadRequest(format!("Module '{}': {}", module.id, e)))? + { + FlowModuleValue::ForloopFlow { modules, .. } + | FlowModuleValue::WhileloopFlow { modules, .. } => { + Self::traverse_modules(&modules, cb)?; + } + FlowModuleValue::BranchOne { branches, default, .. } => { + for branch in branches { + Self::traverse_modules(&branch.modules, cb)?; + } + Self::traverse_modules(&default, cb)?; + } + FlowModuleValue::BranchAll { branches, .. } => { + for branch in branches { + Self::traverse_modules(&branch.modules, cb)?; + } + } + FlowModuleValue::AIAgent { tools, .. } => { + for tool in tools { + match &tool.value { + ToolValue::FlowModule(module_value) => match module_value { + FlowModuleValue::ForloopFlow { modules, .. } + | FlowModuleValue::WhileloopFlow { modules, .. } => { + Self::traverse_modules(&modules, cb)?; + } + FlowModuleValue::BranchOne { branches, default, .. } => { + for branch in branches { + Self::traverse_modules(&branch.modules, cb)?; + } + Self::traverse_modules(&default, cb)?; + } + FlowModuleValue::BranchAll { branches, .. } => { + for branch in branches { + Self::traverse_modules(&branch.modules, cb)?; + } + } + _ => {} + }, + ToolValue::Mcp(_) => { + // MCP tools don't have a FlowModule to traverse + } + } + } + } + _ => {} + } + } + Ok(()) + } } #[derive(Deserialize)] @@ -555,12 +726,108 @@ pub struct Branch { pub parallel: bool, } +// Tool types for AI Agent +#[derive(Serialize, Debug, Clone, Deserialize)] +pub struct AgentTool { + pub id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, + pub value: ToolValue, +} + +// Convert FlowModule -> AgentTool +impl From for AgentTool { + fn from(flow_module: FlowModule) -> Self { + let module_value = serde_json::from_str::(flow_module.value.get()) + .unwrap_or(FlowModuleValue::Identity); + + AgentTool { + id: flow_module.id, + summary: flow_module.summary, + value: ToolValue::FlowModule(module_value), + } + } +} + +// Convert AgentTool -> FlowModule (only for FlowModule type tools) +impl From<&AgentTool> for Option { + fn from(tool: &AgentTool) -> Self { + match &tool.value { + ToolValue::FlowModule(module_value) => Some(FlowModule { + id: tool.id.clone(), + value: to_raw_value(module_value), + summary: tool.summary.clone(), + ..Default::default() + }), + ToolValue::Mcp(_) => None, // MCP tools can't be converted to FlowModule + } + } +} + +#[derive(Serialize, Debug, Clone)] +#[serde(tag = "tool_type", rename_all = "lowercase")] +pub enum ToolValue { + FlowModule(FlowModuleValue), + Mcp(McpToolValue), +} + +// Custom deserializer for backward compatibility with old flows +impl<'de> Deserialize<'de> for ToolValue { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + use serde::de::Error; + + let content = serde_json::Value::deserialize(deserializer)?; + + // First, try to deserialize as the new tagged format (with tool_type field) + #[derive(Deserialize)] + #[serde(tag = "tool_type", rename_all = "lowercase")] + enum TaggedToolValue { + FlowModule(FlowModuleValue), + Mcp(McpToolValue), + } + + if let Ok(tagged) = TaggedToolValue::deserialize(&content) { + return Ok(match tagged { + TaggedToolValue::FlowModule(v) => ToolValue::FlowModule(v), + TaggedToolValue::Mcp(v) => ToolValue::Mcp(v), + }); + } + + // Fall back to legacy format (direct FlowModuleValue without tool_type) + FlowModuleValue::deserialize(&content) + .map(ToolValue::FlowModule) + .map_err(|_| { + D::Error::custom( + "expected ToolValue with tool_type field or legacy FlowModuleValue", + ) + }) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct McpToolValue { + pub resource_path: String, + #[serde(default)] + pub include_tools: Vec, + #[serde(default)] + pub exclude_tools: Vec, +} + +fn is_none_or_empty_vec(expr: &Option>) -> bool +{ + expr.is_none() || expr.as_ref().unwrap().is_empty() +} + #[derive(Serialize, Debug, Clone)] #[serde( tag = "type", rename_all(serialize = "lowercase", deserialize = "lowercase") )] pub enum FlowModuleValue { + /// Reference to another script on the workspace Script { #[serde(default)] #[serde(alias = "input_transform")] @@ -572,13 +839,21 @@ pub enum FlowModuleValue { tag_override: Option, #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pass_flow_input_directly: Option, }, + + /// Reference to another flow on the workspace Flow { #[serde(default)] #[serde(alias = "input_transform")] input_transforms: HashMap, path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pass_flow_input_directly: Option, }, + + /// For loop node ForloopFlow { iterator: InputTransform, modules: Vec, @@ -588,8 +863,10 @@ pub enum FlowModuleValue { skip_failures: bool, parallel: bool, #[serde(skip_serializing_if = "Option::is_none")] - parallelism: Option, + parallelism: Option, }, + + /// While loop node WhileloopFlow { modules: Vec, #[serde(skip_serializing_if = "Option::is_none")] @@ -597,17 +874,24 @@ pub enum FlowModuleValue { #[serde(default = "default_false")] skip_failures: bool, }, + + /// Branch-one node BranchOne { branches: Vec, default: Vec, #[serde(skip_serializing_if = "Option::is_none")] default_node: Option, }, + + /// Branch-all node BranchAll { branches: Vec, #[serde(default = "default_true")] parallel: bool, }, + + /// Inline script node + /// Only exists if parsed from value from `flow_version` | `flow` table. RawScript { #[serde(default)] #[serde(alias = "input_transform", serialize_with = "ordered_map")] @@ -628,11 +912,16 @@ pub enum FlowModuleValue { concurrency_time_window_s: Option, #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing_if = "is_none_or_empty_vec")] assets: Option>, }, + + /// Just a placeholder Identity, - // Internal only, never exposed to the frontend. + + /// Also Inline script node, but instead of being baked into flow, it references `flow_node` + /// Internal only, never exposed to the frontend. + /// Only exists if parsed from value from `flow_version_lite` table. FlowScript { #[serde(default)] #[serde(alias = "input_transform", serialize_with = "ordered_map")] @@ -649,12 +938,14 @@ pub enum FlowModuleValue { concurrency_time_window_s: Option, #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing_if = "is_none_or_empty_vec")] assets: Option>, }, + + // AI agent node AIAgent { input_transforms: HashMap, - tools: Vec, + tools: Vec, }, } @@ -675,7 +966,8 @@ struct UntaggedFlowModuleValue { modules: Option>, skip_failures: Option, parallel: Option, - parallelism: Option, + #[serde(default, deserialize_with = "raw_value_to_input_transform::<_, u16>")] + parallelism: Option, branches: Option>, default: Option>, content: Option, @@ -690,7 +982,8 @@ struct UntaggedFlowModuleValue { default_node: Option, modules_node: Option, assets: Option>, - tools: Option>, + tools: Option>, + pass_flow_input_directly: Option, } impl<'de> Deserialize<'de> for FlowModuleValue { @@ -709,12 +1002,14 @@ impl<'de> Deserialize<'de> for FlowModuleValue { hash: untagged.hash, tag_override: untagged.tag_override, is_trigger: untagged.is_trigger, + pass_flow_input_directly: untagged.pass_flow_input_directly, }), "flow" => Ok(FlowModuleValue::Flow { input_transforms: untagged.input_transforms.unwrap_or_default(), path: untagged .path .ok_or_else(|| serde::de::Error::missing_field("path"))?, + pass_flow_input_directly: untagged.pass_flow_input_directly, }), "forloopflow" => Ok(FlowModuleValue::ForloopFlow { iterator: untagged @@ -856,6 +1151,7 @@ pub fn add_virtual_items_if_necessary(modules: &mut Vec) { continue_on_error: None, skip_if: None, apply_preprocessor: None, + pass_flow_input_directly: None, }); } } diff --git a/backend/windmill-common/src/git_sync_oss.rs b/backend/windmill-common/src/git_sync_oss.rs new file mode 100644 index 0000000000..dcd2abf1f7 --- /dev/null +++ b/backend/windmill-common/src/git_sync_oss.rs @@ -0,0 +1,31 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::git_sync_ee::*; +use url::Url; +#[cfg(not(feature = "private"))] +use sqlx::{Pool, Postgres}; + +#[cfg(not(feature = "private"))] +pub async fn get_github_app_token_internal( + _db: &Pool, + _job_token: &str, +) -> crate::error::Result { + return Err(crate::error::Error::BadRequest("Github app authentication is not available on the open source build".to_string())) +} + +pub fn prepend_token_to_github_url( + github_url: &str, + installation_token: &str, +) -> crate::error::Result { + let url = Url::parse(github_url)?; + + if url.host_str() != Some("github.com") { + return Err(crate::error::Error::BadRequest("Invalid: not a github URL".to_string())); + } + + Ok(format!( + "https://x-access-token:{}@github.com{}", + installation_token, + url.path() + )) +} diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index c26bdc2f7e..13be9ff02f 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -11,6 +11,8 @@ pub const LICENSE_KEY_SETTING: &str = "license_key"; pub const NPM_CONFIG_REGISTRY_SETTING: &str = "npm_config_registry"; pub const BUNFIG_INSTALL_SCOPES_SETTING: &str = "bunfig_install_scopes"; pub const NUGET_CONFIG_SETTING: &str = "nuget_config"; +pub const POWERSHELL_REPO_URL_SETTING: &str = "powershell_repo_url"; +pub const POWERSHELL_REPO_PAT_SETTING: &str = "powershell_repo_pat"; pub const MAVEN_REPOS_SETTING: &str = "maven_repos"; pub const NO_DEFAULT_MAVEN_SETTING: &str = "no_default_maven"; pub const RUBY_REPOS_SETTING: &str = "ruby_repos"; @@ -32,6 +34,7 @@ pub const EXPOSE_DEBUG_METRICS_SETTING: &str = "expose_debug_metrics"; pub const KEEP_JOB_DIR_SETTING: &str = "keep_job_dir"; pub const REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING: &str = "require_preexisting_user_for_oauth"; pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config"; +pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index d6b58896bc..f32b3447c8 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -29,6 +29,14 @@ use crate::{ FlowVersionInfo, ScriptHashInfo, }; +#[derive(Debug, Deserialize, Clone)] +pub struct DynamicInput { + #[serde(rename = "x-windmill-dyn-select-code")] + pub x_windmill_dyn_select_code: String, + #[serde(rename = "x-windmill-dyn-select-lang")] + pub x_windmill_dyn_select_lang: ScriptLang, +} + #[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone)] #[sqlx(type_name = "JOB_TRIGGER_KIND", rename_all = "lowercase")] #[serde(rename_all = "lowercase")] @@ -76,7 +84,7 @@ pub enum JobKind { Dependencies, Flow, FlowPreview, - SingleScriptFlow, + SingleStepFlow, Identity, FlowDependencies, AppDependencies, @@ -92,7 +100,7 @@ impl JobKind { pub fn is_flow(&self) -> bool { matches!( self, - JobKind::Flow | JobKind::FlowPreview | JobKind::SingleScriptFlow | JobKind::FlowNode + JobKind::Flow | JobKind::FlowPreview | JobKind::SingleStepFlow | JobKind::FlowNode ) } @@ -255,6 +263,7 @@ pub struct CompletedJob { pub created_by: String, pub created_at: chrono::DateTime, pub started_at: Option>, + pub completed_at: Option>, pub duration_ms: i64, pub success: bool, #[serde(skip_serializing_if = "Option::is_none")] @@ -315,22 +324,42 @@ impl CompletedJob { #[derive(Debug, Clone)] pub enum JobPayload { + /// Execute Hub Script ScriptHub { path: String, apply_preprocessor: bool, }, + + /// Execute script ScriptHash { hash: ScriptHash, path: String, + /// Override default concurrency key custom_concurrency_key: Option, + /// How many jobs can run at the same time concurrent_limit: Option, + /// In seconds concurrency_time_window_s: Option, + /// If not set, will be inferred from the hash(path + step_id + inputs) + custom_debounce_key: Option, + /// Debouncing delay will be determined by the first job with the key. + /// All subsequent jobs with Some will get debounced. + /// If the job has no delay, it will execute immediately, fully ignoring pending delays. + debounce_delay_s: Option, cache_ttl: Option, dedicated_worker: Option, language: ScriptLang, priority: Option, apply_preprocessor: bool, }, + + /// Execute flow step (can be subflow only). + FlowNode { + id: FlowNodeId, // flow_node(id). + path: String, // flow node inner path (e.g. `outer/branchall-42`). + }, + + /// Execute flow step FlowScript { id: FlowNodeId, // flow_node(id). language: ScriptLang, @@ -341,67 +370,89 @@ pub enum JobPayload { dedicated_worker: Option, path: String, }, - FlowNode { - id: FlowNodeId, // flow_node(id). - path: String, // flow node inner path (e.g. `outer/branchall-42`). - }, + + /// Inline App Script AppScript { id: AppScriptId, // app_script(id). path: Option, language: ScriptLang, cache_ttl: Option, }, + + /// Script/App/FlowAsCode Preview Code(RawCode), + + /// Script Dependency Job Dependencies { path: String, hash: ScriptHash, language: ScriptLang, dedicated_worker: Option, }, + + /// Flow Dependency Job FlowDependencies { path: String, dedicated_worker: Option, version: i64, }, + + /// App Dependency Job AppDependencies { path: String, version: i64, }, + + /// Flow Dependency Job, exposed with API. Requirements can be partially or fully predefined RawFlowDependencies { path: String, flow_value: FlowValue, }, + + /// Dependency Job, exposed with API. Requirements can be predefined RawScriptDependencies { script_path: String, + /// Will reflect raw requirements content (e.g. requirements.txt) content: String, language: ScriptLang, }, + + /// Flow Job Flow { path: String, dedicated_worker: Option, apply_preprocessor: bool, version: i64, }, + RestartedFlow { completed_job_id: Uuid, step_id: String, branch_or_iteration_n: Option, }, + + /// Flow Preview RawFlow { value: FlowValue, path: Option, restarted_from: Option, }, - SingleScriptFlow { + + /// Flow consisting of single script + SingleStepFlow { path: String, - hash: ScriptHash, + hash: Option, + flow_version: Option, args: HashMap>, retry: Option, error_handler_path: Option, error_handler_args: Option>>, + skip_handler: Option, custom_concurrency_key: Option, concurrent_limit: Option, concurrency_time_window_s: Option, + custom_debounce_key: Option, + debounce_delay_s: Option, cache_ttl: Option, priority: Option, tag_override: Option, @@ -418,6 +469,14 @@ pub enum JobPayload { }, } +#[derive(Clone, Serialize, Deserialize, Debug)] +pub struct SkipHandler { + pub path: String, + pub args: HashMap>, + pub stop_condition: String, + pub stop_message: String, +} + #[derive(Clone, Serialize, Deserialize, Debug, Default)] pub struct RawCode { pub content: String, @@ -428,6 +487,8 @@ pub struct RawCode { pub custom_concurrency_key: Option, pub concurrent_limit: Option, pub concurrency_time_window_s: Option, + pub custom_debounce_key: Option, + pub debounce_delay_s: Option, pub cache_ttl: Option, pub dedicated_worker: Option, } @@ -499,6 +560,8 @@ pub async fn script_path_to_payload<'e>( concurrency_key, concurrent_limit, concurrency_time_window_s, + debounce_key, + debounce_delay_s, cache_ttl, language, dedicated_worker, @@ -527,7 +590,9 @@ pub async fn script_path_to_payload<'e>( custom_concurrency_key: concurrency_key, concurrent_limit, concurrency_time_window_s, - cache_ttl: cache_ttl, + custom_debounce_key: debounce_key, + debounce_delay_s, + cache_ttl, language, dedicated_worker, priority, @@ -549,6 +614,11 @@ pub async fn script_path_to_payload<'e>( )) } +#[inline(always)] +pub fn generate_dynamic_input_key(workspace_id: &str, path: &str) -> String { + format!("{workspace_id}:{path}") +} + pub async fn get_payload_tag_from_prefixed_path( path: &str, db: &DB, @@ -752,3 +822,31 @@ pub async fn check_tag_available_for_workspace_internal( return Ok(()); } + +pub async fn lock_debounce_key<'c>( + w_id: &str, + runnable_path: &str, + tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, +) -> error::Result> { + if !*crate::worker::MIN_VERSION_SUPPORTS_DEBOUNCING.read().await { + tracing::warn!("Debouncing is not supported on this version of Windmill. Minimum version required for debouncing support."); + return Ok(None); + } + + let key = format!("{w_id}:{runnable_path}:dependency"); + + tracing::debug!( + workspace_id = %w_id, + runnable_path = %runnable_path, + debounce_key = %key, + "Locking debounce_key for dependency job scheduling" + ); + + sqlx::query_scalar!( + "SELECT job_id FROM debounce_key WHERE key = $1 AND job_id IN (SELECT id FROM v2_job_queue) FOR UPDATE", + &key + ) + .fetch_optional(&mut **tx) + .await + .map_err(error::Error::from) +} diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 7c391a8fee..a4be34aa4b 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -18,7 +18,7 @@ use std::{ }, }; -use tokio::sync::broadcast; +use tokio::{spawn, sync::broadcast}; use ee_oss::CriticalErrorChannel; use error::Error; @@ -43,6 +43,7 @@ pub mod email_ee; pub mod email_oss; pub mod error; pub mod external_ip; +pub mod flow_conversations; pub mod flow_status; pub mod flows; pub mod global_settings; @@ -53,8 +54,12 @@ pub mod job_s3_helpers_ee; #[cfg(feature = "parquet")] pub mod job_s3_helpers_oss; +#[cfg(feature = "private")] +pub mod git_sync_ee; +pub mod git_sync_oss; pub mod jobs; pub mod jwt; +pub mod mcp_client; pub mod more_serde; pub mod oauth2; #[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] @@ -92,6 +97,7 @@ pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5; pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5; pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev"; +pub const PRIVATE_HUB_MIN_VERSION: i32 = 10_000_000; pub const SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs #[macro_export] @@ -152,6 +158,7 @@ lazy_static::lazy_static! { pub static ref DEPLOYED_SCRIPT_HASH_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000); pub static ref FLOW_VERSION_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000); + pub static ref DYNAMIC_INPUT_CACHE: Cache> = Cache::new(1000); pub static ref DEPLOYED_SCRIPT_INFO_CACHE: Cache<(String, i64), ScriptHashInfo> = Cache::new(1000); pub static ref FLOW_INFO_CACHE: Cache<(String, i64), FlowVersionInfo> = Cache::new(1000); @@ -197,8 +204,46 @@ pub async fn shutdown_signal( }, } + spawn(async move { + #[cfg(any(target_os = "linux", target_os = "macos"))] + tokio::select! { + _ = terminate() => { + tracing::error!("2nd shutdown monitor received terminate"); + }, + _ = tokio::signal::ctrl_c() => { + tracing::error!("2nd shutdown monitor received ctrl-c"); + }, + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + tokio::select! { + _ = tokio::signal::ctrl_c() => { + tracing::error!("2nd shutdown monitor received ctrl-c") + }, + } + + tracing::info!("Second terminate signal received, forcefully exiting"); + + let handle = tokio::runtime::Handle::current(); + let metrics = handle.metrics(); + tracing::info!( + "Alive tasks: {}, global queue depth: {}", + metrics.num_alive_tasks(), + metrics.global_queue_depth() + ); + + std::process::exit(1); + }); + tracing::info!("signal received, starting graceful shutdown"); let _ = tx.send(); + + spawn(async move { + tokio::time::sleep(std::time::Duration::from_secs(24 * 7 * 60 * 60)).await; + tracing::info!("Forcefully exiting after 7 days"); + std::process::exit(1); + }); + Ok(()) } @@ -455,7 +500,7 @@ pub struct ExpiringLatestVersionId { expires_at: std::time::Instant, } -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct ScriptHashInfo { pub path: String, pub hash: i64, @@ -463,6 +508,8 @@ pub struct ScriptHashInfo { pub concurrency_key: Option, pub concurrent_limit: Option, pub concurrency_time_window_s: Option, + pub debounce_key: Option, + pub debounce_delay_s: Option, pub cache_ttl: Option, pub language: ScriptLang, pub dedicated_worker: Option, @@ -609,7 +656,7 @@ async fn get_script_info_for_hash_inner<'e, E: sqlx::PgExecutor<'e>>( ) -> error::Result> { let r = sqlx::query_as!( ScriptHashInfo, - "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by, path from script where hash = $1 AND workspace_id = $2", + "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, debounce_key, debounce_delay_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by, path from script where hash = $1 AND workspace_id = $2", hash, w_id ) @@ -623,6 +670,7 @@ pub struct FlowVersionInfo { pub tag: Option, pub early_return: Option, pub has_preprocessor: Option, + pub chat_input_enabled: Option, pub on_behalf_of_email: Option, pub edited_by: String, pub dedicated_worker: Option, @@ -744,7 +792,7 @@ pub fn get_latest_flow_version_info_for_path_from_version< let mut conn = db.acquire().await?; let info = sqlx::query_as!( FlowVersionInfo, - "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by, flow_version.id AS version + "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, (flow_version.value->>'chat_input_enabled')::boolean as chat_input_enabled, on_behalf_of_email, edited_by, flow_version.id AS version FROM flow INNER JOIN flow_version ON flow_version.id = $3 @@ -808,6 +856,8 @@ pub async fn get_latest_hash_for_path<'c, E: sqlx::PgExecutor<'c>>( Option, Option, Option, + Option, + Option, Option, ScriptLang, Option, @@ -817,15 +867,15 @@ pub async fn get_latest_hash_for_path<'c, E: sqlx::PgExecutor<'c>>( String, )> { let r_o = sqlx::query!( - "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, timeout, on_behalf_of_email, created_by FROM script - WHERE path = $1 AND workspace_id = $2 AND archived = false AND (lock IS NOT NULL OR $3 = false) - ORDER BY created_at DESC LIMIT 1", - script_path, - w_id, - require_locked - ) - .fetch_optional(db) - .await?; + "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, debounce_key, debounce_delay_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, timeout, on_behalf_of_email, created_by FROM script + WHERE path = $1 AND workspace_id = $2 AND archived = false AND (lock IS NOT NULL OR $3 = false) + ORDER BY created_at DESC LIMIT 1", + script_path, + w_id, + require_locked + ) + .fetch_optional(db) + .await?; let script = utils::not_found_if_none(r_o, "script", script_path)?; @@ -835,6 +885,8 @@ pub async fn get_latest_hash_for_path<'c, E: sqlx::PgExecutor<'c>>( script.concurrency_key, script.concurrent_limit, script.concurrency_time_window_s, + script.debounce_key, + script.debounce_delay_s, script.cache_ttl, script.language, script.dedicated_worker, diff --git a/backend/windmill-common/src/mcp_client.rs b/backend/windmill-common/src/mcp_client.rs new file mode 100644 index 0000000000..e12aebe5bd --- /dev/null +++ b/backend/windmill-common/src/mcp_client.rs @@ -0,0 +1,228 @@ +use crate::variables::get_secret_value_as_admin; +use crate::DB; +use anyhow::{Context, Result}; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; +use serde_json::{json, Value}; +use std::str::FromStr; + +use rmcp::model::Tool as McpTool; +use rmcp::{ + model::{ + CallToolRequestParam, ClientCapabilities, ClientInfo, Implementation, + InitializeRequestParam, + }, + service::RunningService, + transport::{ + streamable_http_client::StreamableHttpClientTransportConfig, StreamableHttpClientTransport, + }, + RoleClient, ServiceExt, +}; + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +/// MCP server resource configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpResource { + /// Name of the MCP resource (used for prefixing tools) + pub name: String, + /// HTTP URL for the MCP server endpoint + pub url: String, + /// Optional token for authentication + #[serde(skip_serializing_if = "Option::is_none")] + pub token: Option, + /// Optional headers + #[serde(skip_serializing_if = "Option::is_none")] + pub headers: Option>, +} + +/// Metadata for tracking MCP tool sources +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpToolSource { + /// Name of the MCP resource this tool comes from + pub name: String, + /// Original tool name in the MCP server + pub tool_name: String, + /// Path of the MCP resource + pub resource_path: String, +} + +/// MCP client for communicating with external MCP servers +pub struct McpClient { + /// The underlying rmcp client + client: RunningService, + /// Cached list of available tools from the server + available_tools: Vec, +} + +impl McpClient { + /// Create a new MCP client from a resource configuration + pub async fn from_resource(resource: McpResource, db: &DB, w_id: &str) -> Result { + // Build custom reqwest client with headers if provided + let mut headers = HeaderMap::new(); + if let Some(token_path) = &resource.token { + if !token_path.trim().is_empty() { + let value = + get_secret_value_as_admin(db, w_id, token_path.trim_start_matches("$var:")) + .await?; + headers.insert( + HeaderName::from_static("authorization"), + HeaderValue::from_str(format!("Bearer {}", value).as_str())?, + ); + } + } + if let Some(resource_headers) = &resource.headers { + for (key, value) in resource_headers { + match (HeaderName::from_str(key), HeaderValue::from_str(value)) { + (Ok(name), Ok(value)) => { + headers.insert(name, value); + } + _ => { + tracing::warn!("Invalid header: {}={}", key, value); + } + } + } + } + + let reqwest_client = reqwest::Client::builder() + .default_headers(headers) + .build() + .context("Failed to build HTTP client")?; + + // Create the HTTP transport with custom client + let config = StreamableHttpClientTransportConfig::with_uri(resource.url.as_str()); + let transport = StreamableHttpClientTransport::with_client(reqwest_client, config); + + // Set up client info + let client_info = ClientInfo { + protocol_version: Default::default(), + capabilities: ClientCapabilities::default(), + client_info: Implementation { + name: "windmill-ai-agent".to_string(), + title: Some("Windmill AI Agent".to_string()), + version: env!("CARGO_PKG_VERSION").to_string(), + website_url: None, + icons: None, + }, + }; + + // Initialize the connection + let client = client_info + .serve(transport) + .await + .context("Failed to connect to MCP server")?; + + // Immediately fetch available tools + let available_tools = client + .list_tools(Default::default()) + .await + .context("Failed to list tools from MCP server")? + .tools; + + Ok(Self { client, available_tools }) + } + + /// Get the list of available tools from the MCP server + pub fn available_tools(&self) -> &[McpTool] { + &self.available_tools + } + + /// Call a tool on the MCP server, with openai-style arguments + pub async fn call_tool(&self, name: &str, arguments: &str) -> Result { + // Convert OpenAI-style arguments to MCP format + let mcp_args = + Self::openai_args_to_mcp_args(arguments).context("Failed to parse tool arguments")?; + + let result = self + .client + .call_tool(CallToolRequestParam { name: name.to_string().into(), arguments: mcp_args }) + .await + .context(format!("Failed to call MCP tool: {}", name))?; + + // Convert the result to a JSON value + // MCP tools return ToolResult which contains content array + let result_json = + serde_json::to_value(&result).context("Failed to serialize MCP tool result")?; + + Ok(result_json) + } + + /// Close the connection + pub async fn shutdown(self) -> Result<()> { + self.client.cancel().await?; + Ok(()) + } + + /// Fix array schemas to ensure they have the required 'items' property + /// OpenAI requires all array types to have an 'items' field. MCP servers may + /// return schemas without this field, so we add a default. + pub fn fix_array_schemas(schema: &mut Value) { + if let Value::Object(obj) = schema { + // Check if this is an array type + if let Some(type_val) = obj.get("type") { + let is_array = match type_val { + Value::String(s) => s == "array", + Value::Array(arr) => arr.iter().any(|v| v.as_str() == Some("array")), + _ => false, + }; + + // If it's an array and missing 'items', add a default + if is_array && !obj.contains_key("items") { + obj.insert("items".to_string(), json!({})); + } + } + + // Recursively fix nested schemas + if let Some(Value::Object(props)) = obj.get_mut("properties") { + for value in props.values_mut() { + Self::fix_array_schemas(value); + } + } + + // Fix items if present (for nested arrays) + if let Some(items) = obj.get_mut("items") { + Self::fix_array_schemas(items); + } + + // Fix oneOf, anyOf, allOf schemas + for key in &["oneOf", "anyOf", "allOf"] { + if let Some(Value::Array(schemas)) = obj.get_mut(*key) { + for schema in schemas { + Self::fix_array_schemas(schema); + } + } + } + + // Fix additionalProperties if it's a schema + if let Some(additional) = obj.get_mut("additionalProperties") { + if additional.is_object() { + Self::fix_array_schemas(additional); + } + } + } + } + + /// Convert OpenAI-style tool call arguments to MCP format + /// OpenAI sends arguments as a JSON string, MCP expects a Map + fn openai_args_to_mcp_args( + args_str: &str, + ) -> Result>> { + if args_str.trim().is_empty() { + return Ok(None); + } + + let args_value: serde_json::Value = + serde_json::from_str(args_str).context("Failed to parse tool call arguments")?; + + match args_value { + serde_json::Value::Object(map) => Ok(Some(map)), + serde_json::Value::Null => Ok(None), + _ => Ok(Some( + vec![("value".to_string(), args_value)] + .into_iter() + .collect(), + )), + } + } +} diff --git a/backend/windmill-common/src/result_stream.rs b/backend/windmill-common/src/result_stream.rs index 4daa0cbd40..c44c2d556e 100644 --- a/backend/windmill-common/src/result_stream.rs +++ b/backend/windmill-common/src/result_stream.rs @@ -1,5 +1,5 @@ -use uuid::Uuid; use crate::{error, DB}; +use uuid::Uuid; pub const STREAM_PREFIX: &str = "WM_STREAM: "; @@ -14,20 +14,32 @@ pub fn extract_stream_from_logs(line: &str) -> Option { None } - - -pub async fn append_result_stream_db(db: &DB, workspace_id: &str, job_id: &Uuid, nstream: &str) -> error::Result<()> { +pub async fn append_result_stream_db( + db: &DB, + workspace_id: &str, + job_id: &Uuid, + nstream: &str, + offset: i32, +) -> error::Result<()> { if !nstream.is_empty() { sqlx::query!( r#" - INSERT INTO job_result_stream (workspace_id, job_id, stream) - VALUES ($1, $2, $3) - ON CONFLICT (job_id) DO UPDATE SET stream = job_result_stream.stream || $3 + INSERT INTO job_result_stream_v2 (workspace_id, job_id, stream, idx) + VALUES ( + $1, + $2, + $3, + $4 + ) + ON CONFLICT (job_id, idx) DO UPDATE SET stream = job_result_stream_v2.stream || EXCLUDED.stream "#, workspace_id, job_id, nstream, - ).execute(db).await?; + offset + ) + .execute(db) + .await?; } Ok(()) } diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 7ffcad3cc8..899e3a2fd5 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -1,4 +1,3 @@ -use crate::db::Authed; use crate::error::{self}; #[cfg(feature = "parquet")] use aws_sdk_sts::config::ProvideCredentials; @@ -15,9 +14,12 @@ use object_store::gcp::GoogleCloudStorageBuilder; use object_store::ObjectStore; #[cfg(feature = "parquet")] use object_store::{aws::AmazonS3Builder, ClientOptions}; +use quick_cache::sync::Cache; #[cfg(feature = "parquet")] use reqwest::header::HeaderMap; -use serde::{Deserialize, Serialize}; +use serde::de::Visitor; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::fmt; #[cfg(feature = "parquet")] use std::sync::{Arc, Mutex}; @@ -216,7 +218,7 @@ pub async fn reload_object_store_setting(db: &crate::DB) -> ObjectStoreReload { return ObjectStoreReload::Never; } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(tag = "type")] pub enum LargeFileStorage { S3Storage(S3Storage), @@ -247,27 +249,106 @@ impl LargeFileStorage { } .unwrap_or(false) } + pub fn get_advanced_permissions(&self) -> Option<&Vec> { + match self { + LargeFileStorage::S3Storage(lfs) => lfs.advanced_permissions.as_ref(), + LargeFileStorage::S3AwsOidc(lfs) => lfs.advanced_permissions.as_ref(), + LargeFileStorage::AzureBlobStorage(lfs) => lfs.advanced_permissions.as_ref(), + LargeFileStorage::AzureWorkloadIdentity(lfs) => lfs.advanced_permissions.as_ref(), + LargeFileStorage::GoogleCloudStorage(glfs) => glfs.advanced_permissions.as_ref(), + } + } } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct S3Storage { pub s3_resource_path: String, #[serde(skip_serializing_if = "Option::is_none")] pub public_resource: Option, + pub advanced_permissions: Option>, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct AzureBlobStorage { pub azure_blob_resource_path: String, #[serde(skip_serializing_if = "Option::is_none")] pub public_resource: Option, + pub advanced_permissions: Option>, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct GoogleCloudStorage { pub gcs_resource_path: String, #[serde(skip_serializing_if = "Option::is_none")] pub public_resource: Option, + pub advanced_permissions: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] +pub struct S3PermissionRule { + pub pattern: String, + pub allow: S3Permission, // read, write, delete, list +} +bitflags::bitflags! { + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub struct S3Permission: u8 { + const READ = 0b0001; + const WRITE = 0b0010; + const DELETE = 0b0100; + const LIST = 0b1000; + } +} + +impl Serialize for S3Permission { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut perms = Vec::new(); + if self.contains(S3Permission::READ) { + perms.push("read"); + } + if self.contains(S3Permission::WRITE) { + perms.push("write"); + } + if self.contains(S3Permission::DELETE) { + perms.push("delete"); + } + if self.contains(S3Permission::LIST) { + perms.push("list"); + } + let perms = perms.join(","); + perms.serialize(serializer) + } +} +impl<'de> Deserialize<'de> for S3Permission { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct PermVisitor; + impl<'de> Visitor<'de> for PermVisitor { + type Value = S3Permission; + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("comma separated list of permissions: read, write, delete, list") + } + fn visit_str(self, v: &str) -> Result { + let mut perms = S3Permission::empty(); + for value in v.split(',') { + perms |= match value { + "read" => S3Permission::READ, + "write" => S3Permission::WRITE, + "delete" => S3Permission::DELETE, + "list" => S3Permission::LIST, + _ => S3Permission::empty(), // ignore unknown permissions + }; + } + Ok(perms) + } + } + + deserializer.deserialize_str(PermVisitor) + } } #[derive(Clone, Debug)] @@ -334,18 +415,6 @@ pub struct AzureBlobResource { pub federated_token_file: Option, } -impl AzureBlobResource { - pub fn get_endpoint_url(&self) -> error::Result { - Ok(render_endpoint( - self.endpoint.clone().unwrap_or_else(|| "".to_string()), - self.use_ssl.unwrap_or(false), - None, - None, - "".to_string(), - )) - } -} - fn as_string<'de, D>(deserializer: D) -> Result where D: serde::de::Deserializer<'de>, @@ -446,6 +515,68 @@ pub async fn build_object_store_client( } } +#[derive(PartialEq)] +pub enum BundleFormat { + Esm, + Cjs, +} + +impl BundleFormat { + pub fn from_string(s: &str) -> Option { + match s { + "esm" => Some(Self::Esm), + "cjs" => Some(Self::Cjs), + _ => None, + } + } +} + +pub async fn upload_artifact_to_store( + path: &str, + data: bytes::Bytes, + standalone_dir: &str, +) -> error::Result<()> { + #[cfg(all(feature = "enterprise", feature = "parquet"))] + let object_store = crate::s3_helpers::get_object_store().await; + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] + let object_store: Option<()> = None; + Ok( + if &crate::utils::MODE_AND_ADDONS.mode == &crate::utils::Mode::Standalone + && object_store.is_none() + { + let path = format!("{}/{}", standalone_dir, path); + tracing::info!("Writing file to path {path}"); + + let split_path = path.split("/").collect::>(); + std::fs::create_dir_all(split_path[..split_path.len() - 1].join("/"))?; + + crate::worker::write_file_bytes(&path, &data)?; + } else { + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] + { + return Err(error::Error::ExecutionErr( + "codebase is an EE feature".to_string(), + )); + } + + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if let Some(os) = object_store { + if let Err(e) = os + .put(&object_store::path::Path::from(path), data.into()) + .await + { + tracing::info!("Failed to put snapshot to s3 at {path}: {:?}", e); + return Err(error::Error::ExecutionErr(format!( + "Failed to put {path} to s3" + ))); + } + } else { + return Err(error::Error::BadConfig("Object store is required for snapshot script and is not configured for servers".to_string())); + } + }, + ) +} + #[cfg(feature = "parquet")] pub async fn attempt_fetch_bytes( client: Arc, @@ -1105,43 +1236,11 @@ pub fn duckdb_connection_settings_internal( return Ok(response); } -impl ObjectStoreResource { - pub fn get_endpoint_url(&self) -> error::Result { - match self { - ObjectStoreResource::S3(s3_resource) => Ok(render_endpoint( - s3_resource.endpoint.clone(), - s3_resource.use_ssl, - s3_resource.port, - s3_resource.path_style, - s3_resource.bucket.clone(), - )), - ObjectStoreResource::Gcs(gcs_resource) => Ok(format!( - "https://storage.googleapis.com/{}", - gcs_resource.bucket - )), - ObjectStoreResource::Azure(az_resource) => az_resource.get_endpoint_url(), - } - } -} - -pub fn check_lfs_object_path_permissions( - lfs: &LargeFileStorage, - _object_path: &str, - authed: &Authed, -) -> error::Result<()> { - if authed.is_admin || lfs.is_public_resource() { - return Ok(()); - } - let _username = authed.username.as_str(); - - // TODO : Extend permission possibilities - - // if lfs.restrict_to_user_paths() { - // if !object_path.starts_with(&format!("u/{username}/")) { - // return Err(error::Error::NotAuthorized(format!( - // "Can only access paths u/{username}/**" - // ))); - // } - // } - return Ok(()); +// DuckDB does not parse anything in case of S3 errors and just returns a generic error message. +// To display better error messages, we cache the errors in a Map +// +// We leverage the fact that workers have an internal server to insert the error message +// from the S3 Proxy, and read it directly in memory from the worker. +lazy_static::lazy_static! { + pub static ref S3_PROXY_LAST_ERRORS_CACHE: Cache = Cache::new(4); } diff --git a/backend/windmill-common/src/schedule.rs b/backend/windmill-common/src/schedule.rs index e78d3a211c..cc7c9af758 100644 --- a/backend/windmill-common/src/schedule.rs +++ b/backend/windmill-common/src/schedule.rs @@ -60,6 +60,8 @@ pub struct Schedule { pub paused_until: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub cron_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub dynamic_skip: Option, } impl Schedule { diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 2a9981d01f..34ba5a1694 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -9,6 +9,7 @@ use std::{ fmt::{self, Display}, hash::{Hash, Hasher}, + ops::Deref, str::FromStr, }; @@ -16,7 +17,7 @@ use crate::{ assets::AssetWithAltAccessType, error::{to_anyhow, Error}, utils::http_get_from_hub, - DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, + DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, PRIVATE_HUB_MIN_VERSION, }; use crate::worker::HUB_CACHE_DIR; @@ -131,12 +132,25 @@ impl FromStr for ScriptLang { #[sqlx(transparent)] pub struct ScriptHash(pub i64); +impl Deref for ScriptHash { + type Target = i64; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + impl Into for ScriptHash { fn into(self) -> u64 { self.0 as u64 } } +impl From for ScriptHash { + fn from(value: i64) -> Self { + Self(value) + } +} + #[derive(PartialEq, sqlx::Type)] #[sqlx(transparent, no_pg_array)] pub struct ScriptHashes(pub Vec); @@ -160,7 +174,10 @@ impl<'de> Deserialize<'de> for ScriptHash { D: Deserializer<'de>, { let s = String::deserialize(deserializer)?; - let i = to_i64(&s).map_err(|e| D::Error::custom(format!("{}", e)))?; + let i = to_i64(&s).map_err(|e| { + tracing::error!("Could not deserialize ScriptHash. Note, input should be in Hex and digit amount should be divisible by 16 (can be padded). err: {}", &e); + D::Error::custom(format!("{}", e)) + })?; Ok(ScriptHash(i)) } } @@ -202,9 +219,54 @@ impl Display for ScriptKind { } } -pub const PREVIEW_IS_CODEBASE_HASH: i64 = -42; -pub const PREVIEW_IS_TAR_CODEBASE_HASH: i64 = -43; +const PREVIEW_IS_CODEBASE_HASH: i64 = -42; +const PREVIEW_IS_TAR_CODEBASE_HASH: i64 = -43; +const PREVIEW_IS_ESM_CODEBASE_HASH: i64 = -44; +const PREVIEW_IS_TAR_ESM_CODEBASE_HASH: i64 = -45; +pub fn is_special_codebase_hash(hash: i64) -> bool { + hash == PREVIEW_IS_CODEBASE_HASH + || hash == PREVIEW_IS_TAR_CODEBASE_HASH + || hash == PREVIEW_IS_ESM_CODEBASE_HASH + || hash == PREVIEW_IS_TAR_ESM_CODEBASE_HASH +} + +pub fn codebase_to_hash(is_tar: bool, is_esm: bool) -> i64 { + if is_tar { + if is_esm { + PREVIEW_IS_TAR_ESM_CODEBASE_HASH + } else { + PREVIEW_IS_TAR_CODEBASE_HASH + } + } else { + if is_esm { + PREVIEW_IS_ESM_CODEBASE_HASH + } else { + PREVIEW_IS_CODEBASE_HASH + } + } +} + +pub fn hash_to_codebase_id(job_id: &str, hash: i64) -> Option { + match hash { + PREVIEW_IS_CODEBASE_HASH => Some(job_id.to_string()), + PREVIEW_IS_TAR_CODEBASE_HASH => Some(format!("{}.tar", job_id)), + PREVIEW_IS_ESM_CODEBASE_HASH => Some(format!("{}.esm", job_id)), + PREVIEW_IS_TAR_ESM_CODEBASE_HASH => Some(format!("{}.esm.tar", job_id)), + _ => None, + } +} + +pub struct CodebaseInfo { + pub is_tar: bool, + pub is_esm: bool, +} + +pub fn id_to_codebase_info(id: &str) -> CodebaseInfo { + let is_tar = id.ends_with(".tar"); + let is_esm = id.contains(".esm"); + CodebaseInfo { is_tar, is_esm } +} #[derive(Serialize, sqlx::FromRow)] pub struct Script { pub workspace_id: String, @@ -232,10 +294,16 @@ pub struct Script { #[serde(skip_serializing_if = "Option::is_none")] pub envs: Option>, #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub concurrent_limit: Option, #[serde(skip_serializing_if = "Option::is_none")] pub concurrency_time_window_s: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub debounce_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub debounce_delay_s: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub dedicated_worker: Option, #[serde(skip_serializing_if = "Option::is_none")] pub ws_error_handler_muted: Option, @@ -250,8 +318,6 @@ pub struct Script { #[serde(skip_serializing_if = "Option::is_none")] pub restart_unless_cancelled: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency_key: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub visible_to_runner_only: Option, #[serde(skip_serializing_if = "Option::is_none")] pub no_main_func: Option, @@ -329,7 +395,7 @@ impl Hash for Schema { } } -#[derive(Serialize, Deserialize, Hash)] +#[derive(Serialize, Deserialize, Hash, Debug)] pub struct NewScript { pub path: String, pub parent_hash: Option, @@ -346,8 +412,13 @@ pub struct NewScript { pub tag: Option, pub draft_only: Option, pub envs: Option>, + pub concurrency_key: Option, pub concurrent_limit: Option, pub concurrency_time_window_s: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub debounce_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub debounce_delay_s: Option, pub cache_ttl: Option, pub dedicated_worker: Option, pub ws_error_handler_muted: Option, @@ -357,7 +428,6 @@ pub struct NewScript { pub restart_unless_cancelled: Option, pub deployment_message: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency_key: Option, pub visible_to_runner_only: Option, pub no_main_func: Option, pub codebase: Option, @@ -493,6 +563,7 @@ pub async fn get_hub_script_by_path( let hub_base_url = HUB_BASE_URL.read().await.clone(); + // let result = http_get_from_hub( http_client, &format!("{}/raw/{}.ts", hub_base_url, path), @@ -514,7 +585,7 @@ pub async fn get_hub_script_by_path( && path .split("/") .next() - .is_some_and(|x| x.parse::().is_ok_and(|x| x < 10_000_000)) + .is_some_and(|x| x.parse::().is_ok_and(|x| x < PRIVATE_HUB_MIN_VERSION)) { tracing::info!( "Not found on private hub, fallback to default hub for {}", @@ -599,10 +670,9 @@ async fn get_full_hub_script_by_path_inner( Ok(response) => Ok(response), Err(e) => { if hub_base_url != DEFAULT_HUB_BASE_URL - && path - .split("/") - .next() - .is_some_and(|x| x.parse::().is_ok_and(|x| x < 10_000_000)) + && path.split("/").next().is_some_and(|x| { + x.parse::().is_ok_and(|x| x < PRIVATE_HUB_MIN_VERSION) + }) { // TODO: should only fallback to default hub if status is 404 (hub returns 500 currently) tracing::info!( @@ -661,3 +731,93 @@ pub fn hash_script(ns: &NewScript) -> i64 { ns.hash(&mut dh); dh.finish() as i64 } + +pub async fn clone_script<'c>( + base_hash: ScriptHash, + w_id: &str, + deployment_message: Option, + tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, +) -> crate::error::Result { + let s = + sqlx::query_as::<_, Script>("SELECT * FROM script WHERE hash = $1 AND workspace_id = $2") + .bind(base_hash.0) + .bind(w_id) + .fetch_one(&mut **tx) + .await?; + + let ns = NewScript { + path: s.path.clone(), + parent_hash: Some(base_hash), + summary: s.summary, + description: s.description, + content: s.content, + schema: s.schema, + is_template: Some(s.is_template), + // TODO: Make it either None everywhere (particularly when raw reqs are calculated) + // Or handle this case and conditionally make Some (only with raw reqs) + lock: None, + language: s.language, + kind: Some(s.kind), + tag: s.tag, + draft_only: s.draft_only, + envs: s.envs, + concurrent_limit: s.concurrent_limit, + concurrency_time_window_s: s.concurrency_time_window_s, + cache_ttl: s.cache_ttl, + dedicated_worker: s.dedicated_worker, + ws_error_handler_muted: s.ws_error_handler_muted, + priority: s.priority, + timeout: s.timeout, + delete_after_use: s.delete_after_use, + restart_unless_cancelled: s.restart_unless_cancelled, + deployment_message, + concurrency_key: s.concurrency_key, + visible_to_runner_only: s.visible_to_runner_only, + no_main_func: s.no_main_func, + codebase: s.codebase, + has_preprocessor: s.has_preprocessor, + on_behalf_of_email: s.on_behalf_of_email, + assets: s.assets, + debounce_delay_s: s.debounce_delay_s, + debounce_key: s.debounce_key, + }; + + let new_hash = hash_script(&ns); + + tracing::debug!( + "cloning script at path {} from '{}' to '{}'", + s.path, + *base_hash, + new_hash + ); + + sqlx::query!(" + INSERT INTO script + (workspace_id, hash, path, parent_hashes, summary, description, content, \ + created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ + draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ + dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ + delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, \ + codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s) + + SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, \ + content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, \ + draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ + dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ + delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, \ + codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s + + FROM script WHERE hash = $2 AND workspace_id = $3; + ", new_hash, base_hash.0, w_id).execute(&mut **tx).await?; + + // Archive base. + sqlx::query!( + "UPDATE script SET archived = true WHERE hash = $1 AND workspace_id = $2", + *base_hash, + w_id + ) + .execute(&mut **tx) + .await?; + + Ok(new_hash) +} diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 77194ac0a9..e02281ea58 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -52,6 +52,8 @@ pub fn initialize_tracing( let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into()); let rust_log_env = std::env::var("RUST_LOG"); + let rust_log_stdout_env = std::env::var("RUST_LOG_STDOUT"); + if rust_log_env .as_ref() .is_ok_and(|x| x == "debug" || x == "info") @@ -95,59 +97,115 @@ pub fn initialize_tracing( let (log_file_writer, _guard) = NonBlockingBuilder::default() .lossy(false) .finish(file_appender); - let stdout_and_log_file_writer = std::io::stdout.and(log_file_writer); // let job_logs_filter = tracing_subscriber::filter::Targets::new() // .with_target("windmill:job_log", tracing::Level::TRACE); - let env_filter = EnvFilter::builder() + // Create the base filter for file writer (always uses RUST_LOG) + let file_env_filter = EnvFilter::builder() .with_default_directive(tracing::level_filters::LevelFilter::ERROR.into()) .from_env_lossy(); - let ts_base = tracing_subscriber::registry().with(env_filter); + // Create the filter for stdout (uses RUST_LOG_STDOUT if available, otherwise RUST_LOG) + let stdout_env_filter = if rust_log_stdout_env.is_ok() { + // Temporarily set RUST_LOG to RUST_LOG_STDOUT value to parse it + let original_rust_log = std::env::var("RUST_LOG").ok(); + std::env::set_var("RUST_LOG", rust_log_stdout_env.unwrap()); + let filter = EnvFilter::builder() + .with_default_directive(tracing::level_filters::LevelFilter::ERROR.into()) + .from_env_lossy(); + // Restore original RUST_LOG + match original_rust_log { + Some(val) => std::env::set_var("RUST_LOG", val), + None => std::env::remove_var("RUST_LOG"), + } + filter + } else { + file_env_filter.clone() + }; + + // Create a common filter for OTEL logs bridge and tracing layer to respect RUST_LOG + let otel_logs_filter = file_env_filter.clone(); + + // Apply filter to the opentelemetry tracing layer to prevent debug events from being attached to spans + #[cfg(all(feature = "otel", feature = "enterprise"))] + let opentelemetry_filtered = { + let otel_tracing_filter = file_env_filter.clone(); + opentelemetry.map(|layer| layer.with_filter(otel_tracing_filter)) + }; + + #[cfg(not(all(feature = "otel", feature = "enterprise")))] + let opentelemetry_filtered = opentelemetry; + + let base_layer = tracing_subscriber::registry() + .with(logs_bridge.with_filter(otel_logs_filter)) + .with(opentelemetry_filtered); match *JSON_FMT { - true => ts_base - .with(logs_bridge) - .with(opentelemetry) - // .with(env_filter2.add_directive("windmill:job_log=off".parse().unwrap())) - .with( - json_layer() - .with_writer(stdout_and_log_file_writer) - .flatten_event(true) - .with_filter( - Targets::new() - .with_target( - "windmill:job_log", - tracing::level_filters::LevelFilter::OFF, - ) - .with_default(default_env_filter), - ), - ) - .with(CountingLayer::new()) - .init(), - false => ts_base - .with(logs_bridge) - .with(opentelemetry) - // .with(env_filter2.add_directive("windmill:job_log=off".parse().unwrap())) - .with( - compact_layer() - .with_writer(stdout_and_log_file_writer) - .with_ansi(style.to_lowercase() != "never") - .with_file(true) - .with_line_number(true) - .with_target(false) - .with_filter( - Targets::new() - .with_target( - "windmill:job_log", - tracing::level_filters::LevelFilter::OFF, - ) - .with_default(default_env_filter), - ), - ) - .with(CountingLayer::new()) - .init(), + true => { + // Stdout layer with its own filter + let stdout_layer = json_layer() + .with_writer(std::io::stdout) + .flatten_event(true) + .with_filter(stdout_env_filter) + .with_filter( + Targets::new() + .with_target("windmill:job_log", tracing::level_filters::LevelFilter::OFF) + .with_default(default_env_filter), + ); + + // File layer with its own filter + let file_layer = json_layer() + .with_writer(log_file_writer) + .flatten_event(true) + .with_filter(file_env_filter) + .with_filter( + Targets::new() + .with_target("windmill:job_log", tracing::level_filters::LevelFilter::OFF) + .with_default(default_env_filter), + ); + + base_layer + .with(stdout_layer) + .with(file_layer) + .with(CountingLayer::new()) + .init() + } + false => { + // Stdout layer with its own filter + let stdout_layer = compact_layer() + .with_writer(std::io::stdout) + .with_ansi(style.to_lowercase() != "never") + .with_file(true) + .with_line_number(true) + .with_target(false) + .with_filter(stdout_env_filter) + .with_filter( + Targets::new() + .with_target("windmill:job_log", tracing::level_filters::LevelFilter::OFF) + .with_default(default_env_filter), + ); + + // File layer with its own filter + let file_layer = compact_layer() + .with_writer(log_file_writer) + .with_ansi(false) // No ANSI codes in log files + .with_file(true) + .with_line_number(true) + .with_target(false) + .with_filter(file_env_filter) + .with_filter( + Targets::new() + .with_target("windmill:job_log", tracing::level_filters::LevelFilter::OFF) + .with_default(default_env_filter), + ); + + base_layer + .with(stdout_layer) + .with(file_layer) + .with(CountingLayer::new()) + .init() + } } (_guard, meter_provider) } diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 771678bb2f..174d3cd261 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -28,8 +28,10 @@ use sha2::{Digest, Sha256}; use sqlx::{Pool, Postgres}; use std::borrow::Cow; use std::fmt::Display; +use std::sync::Arc; use std::{fs::DirBuilder as SyncDirBuilder, str::FromStr}; use tokio::fs::DirBuilder as AsyncDirBuilder; +use tokio::sync::RwLock; use url::Url; pub const MAX_PER_PAGE: usize = 10000; @@ -49,12 +51,23 @@ use std::sync::atomic::Ordering; use crate::worker::CLOUD_HOSTED; lazy_static::lazy_static! { + pub static ref FORCE_IPV4: bool = std::env::var("FORCE_IPV4") + .map(|v| v.to_lowercase() == "true" || v == "1") + .unwrap_or(false); - pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new() - .user_agent("windmill/beta") - .timeout(std::time::Duration::from_secs(20)) - .connect_timeout(std::time::Duration::from_secs(10)) - .build().unwrap(); + pub static ref HTTP_CLIENT: Client = { + let mut builder = reqwest::ClientBuilder::new() + .user_agent("windmill/beta") + .timeout(std::time::Duration::from_secs(20)) + .connect_timeout(std::time::Duration::from_secs(10)); + + if *FORCE_IPV4 { + tracing::info!("FORCE_IPV4 is enabled - HTTP client will only use IPv4"); + builder = builder.local_address(std::net::IpAddr::V4(std::net::Ipv4Addr::new(0, 0, 0, 0))); + } + + builder.build().unwrap() + }; pub static ref GIT_SEM_VERSION: Version = Version::parse( if GIT_VERSION.starts_with('v') { &GIT_VERSION[1..] @@ -134,6 +147,8 @@ lazy_static::lazy_static! { mode, } }; + + pub static ref HUB_API_SECRET: Arc>> = Arc::new(RwLock::new(None)); } lazy_static::lazy_static! { @@ -173,6 +188,18 @@ pub fn require_admin(is_admin: bool, username: &str) -> Result<()> { } } +/// Configure reqwest::ClientBuilder with environment-based settings +/// When FORCE_IPV4=true environment variable is set, this configures the client +/// to only use IPv4 addresses by binding to 0.0.0.0 +pub fn configure_client(builder: reqwest::ClientBuilder) -> reqwest::ClientBuilder { + if *FORCE_IPV4 { + tracing::info!("FORCE_IPV4 is enabled - HTTP client will only use IPv4"); + builder.local_address(std::net::IpAddr::V4(std::net::Ipv4Addr::new(0, 0, 0, 0))) + } else { + builder + } +} + pub async fn require_admin_or_devops( is_admin: bool, username: &str, @@ -246,6 +273,7 @@ pub async fn now_from_db<'c, E: sqlx::PgExecutor<'c>>( ) -> Result> { Ok(sqlx::query_scalar!("SELECT now()") .fetch_one(db) + .warn_after_seconds_with_sql(1, "now_from_db".to_string()) .await? .unwrap()) } @@ -333,6 +361,10 @@ pub async fn http_get_from_hub( request = request.header("X-uid", uid); } + if let Some(hub_api_secret) = HUB_API_SECRET.read().await.clone() { + request = request.header("X-api-secret", hub_api_secret); + } + if let Some(query_params) = query_params { for (key, value) in query_params { request = request.query(&[(key, value)]); @@ -909,3 +941,9 @@ mod tests { assert_eq!(r, "host=localhost port=5432 user=postgres dbname=test_db"); } } + +#[derive(Clone)] +pub struct ExpiringCacheEntry { + pub value: T, + pub expiry: std::time::Instant, +} diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index ae81512a0a..9dd9edc64d 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -6,7 +6,7 @@ * LICENSE-AGPL for a copy of the license. */ -use crate::error; +use crate::error::{self, Error}; use crate::scripts::ScriptHash; use crate::utils::WarnAfterExt; use crate::worker::Connection; @@ -164,7 +164,12 @@ pub async fn get_secret_value_as_admin( let value = variable.value; if !value.is_empty() { let mc = build_crypt(db, w_id).await?; - decrypt(&mc, value)? + decrypt(&mc, value).map_err(|e| { + crate::error::Error::internal_err(format!( + "Error decrypting variable {}: {}", + variable.path, e + )) + })? } else { "".to_string() } @@ -214,6 +219,7 @@ pub async fn get_reserved_variables( root_job_id: Option, scheduled_for: Option>, runnable_id: Option, + end_user_email: Option, ) -> Vec { let state_path = { let trigger = if schedule_path.is_some() { @@ -384,6 +390,12 @@ pub async fn get_reserved_variables( description: "Hash of the script. Useful as cache key for cache that should be runnable specific.".to_string(), is_custom: false, }, + ContextualVariable { + name: "WM_END_USER_EMAIL".to_string(), + value: end_user_email.unwrap_or_else(|| "".to_string()), + description: "Email of the end user that executed the current script. Only available when triggered from an app.".to_string(), + is_custom: false, + }, ].into_iter().chain(custom_envs.into_iter().map(|(name, value)| ContextualVariable { name, value, @@ -426,3 +438,37 @@ async fn get_cached_workspace_envs(conn: &Connection, w_id: &str) -> Vec<(String }; custom_envs } + +pub async fn get_variable_or_self(path: String, db: &DB, w_id: &str) -> crate::error::Result { + if !path.starts_with("$var:") { + return Ok(path); + } + let path = path.strip_prefix("$var:").unwrap().to_string(); + + let record = sqlx::query!( + "SELECT value, is_secret + FROM variable + WHERE path = $1 AND workspace_id = $2", + &path, + &w_id + ) + .fetch_optional(db) + .await?; + + if let Some(record) = record { + let mut value = record.value; + if record.is_secret { + let mc = build_crypt(db, w_id).await?; + value = decrypt(&mc, value).map_err(|e| { + Error::internal_err(format!("Error decrypting variable {}: {}", path, e)) + })?; + } + + Ok(value) + } else { + Err(Error::NotFound(format!( + "Variable not found when resolving `$var:{}`", + path + ))) + } +} diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 1254bcb04d..eafd8cd894 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -14,6 +14,7 @@ use std::{ collections::{HashMap, HashSet}, fs::{self, File}, io::Write, + ops::Deref, panic::Location, path::{Component, Path, PathBuf}, str::FromStr, @@ -167,6 +168,7 @@ lazy_static::lazy_static! { "powershell".to_string(), "nativets".to_string(), "mysql".to_string(), + "oracledb".to_string(), "bun".to_string(), "postgresql".to_string(), "bigquery".to_string(), @@ -187,6 +189,18 @@ lazy_static::lazy_static! { "other".to_string() ]; + pub static ref NATIVE_TAGS: Vec = vec![ + "nativets".to_string(), + "postgresql".to_string(), + "mysql".to_string(), + "graphql".to_string(), + "snowflake".to_string(), + "mssql".to_string(), + "bigquery".to_string(), + "oracledb".to_string() + // for related places search: ADD_NEW_LANG + ]; + pub static ref DEFAULT_TAGS_PER_WORKSPACE: AtomicBool = AtomicBool::new(false); pub static ref DEFAULT_TAGS_WORKSPACES: Arc>>> = Arc::new(RwLock::new(None)); @@ -250,6 +264,10 @@ lazy_static::lazy_static! { .unwrap_or(false); pub static ref MIN_VERSION: Arc> = Arc::new(RwLock::new(Version::new(0, 0, 0))); + /// Global flag indicating if all workers support the debouncing feature (>= 1.566.0) + /// Debouncing consolidates multiple dependency job requests within a time window to avoid redundant work + /// This flag is updated during worker initialization by checking the minimum version across all workers + pub static ref MIN_VERSION_SUPPORTS_DEBOUNCING: Arc> = Arc::new(RwLock::new(false)); pub static ref MIN_VERSION_IS_AT_LEAST_1_461: Arc> = Arc::new(RwLock::new(false)); pub static ref MIN_VERSION_IS_AT_LEAST_1_427: Arc> = Arc::new(RwLock::new(false)); pub static ref MIN_VERSION_IS_AT_LEAST_1_432: Arc> = Arc::new(RwLock::new(false)); @@ -258,7 +276,7 @@ lazy_static::lazy_static! { // Features flags: pub static ref DISABLE_FLOW_SCRIPT: bool = std::env::var("DISABLE_FLOW_SCRIPT").ok().is_some_and(|x| x == "1" || x == "true"); - pub static ref ROOT_STANDALONE_BUNDLE_DIR: String = format!("{}/.windmill/standalone_bundle/", std::env::var("HOME").unwrap_or_else(|_| "/root".to_string())); + pub static ref ROOT_STANDALONE_BUNDLE_DIR: String = format!("{}/.windmill/standalone_bundle", std::env::var("HOME").unwrap_or_else(|_| "/root".to_string())); } pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/"); @@ -266,7 +284,18 @@ pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/"); pub static MIN_VERSION_IS_LATEST: AtomicBool = AtomicBool::new(false); #[derive(Clone)] -pub struct HttpClient(pub ClientWithMiddleware); +pub struct HttpClient { + pub client: ClientWithMiddleware, + pub base_internal_url: Option, +} + +impl Deref for HttpClient { + type Target = ClientWithMiddleware; + + fn deref(&self) -> &Self::Target { + &self.client + } +} impl HttpClient { pub async fn post( @@ -275,10 +304,12 @@ impl HttpClient { headers: Option, body: &T, ) -> anyhow::Result { - let response_builder = self - .0 - .post(format!("{}{}", *BASE_INTERNAL_URL, url)) - .json(body); + let base_url = self + .base_internal_url + .clone() + .unwrap_or(BASE_INTERNAL_URL.clone().to_owned()); + + let response_builder = self.client.post(format!("{}{}", base_url, url)).json(body); let response_builder = match headers { Some(headers) => response_builder.headers(headers), @@ -302,9 +333,14 @@ impl HttpClient { } pub async fn get(&self, url: &str) -> anyhow::Result { + let base_url = self + .base_internal_url + .clone() + .unwrap_or(BASE_INTERNAL_URL.clone().to_owned()); + let response = self - .0 - .get(format!("{}{}", *BASE_INTERNAL_URL, url)) + .client + .get(format!("{}{}", base_url, url)) .send() .await .map_err(|e| anyhow::anyhow!(e))?; @@ -386,6 +422,10 @@ fn format_pull_query(peek: String) -> String { raw_flow, script_entrypoint_override, preprocessed FROM v2_job WHERE id = (SELECT id FROM peek) + ), delete_debounce AS NOT MATERIALIZED ( + DELETE FROM debounce_key + USING j + WHERE j.kind::text != 'flowdependencies' AND j.kind::text != 'appdependencies' AND j.kind::text != 'dependencies' AND debounce_key.job_id = j.id ) SELECT j.id, j.workspace_id, j.parent_job, j.created_by, started_at, scheduled_for, j.runnable_id, j.runnable_path, j.args, canceled_by, canceled_reason, j.kind, j.trigger, j.trigger_kind, j.permissioned_as, @@ -395,11 +435,12 @@ fn format_pull_query(peek: String) -> String { j.timeout, j.flow_step_id, j.cache_ttl, j.priority, j.raw_code, j.raw_lock, j.raw_flow, j.script_entrypoint_override, j.preprocessed, pj.runnable_path as parent_runnable_path, COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin, - p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders + p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email FROM q, j LEFT JOIN v2_job_status f USING (id) LEFT JOIN job_perms p ON p.job_id = j.id - LEFT JOIN v2_job pj ON j.parent_job = pj.id", + LEFT JOIN v2_job pj ON j.parent_job = pj.id + ", peek ); // tracing::debug!("pull query: {}", r); @@ -457,6 +498,7 @@ pub async fn store_pull_query(wc: &WorkerConfig) { pub const TMP_DIR: &str = "/tmp/windmill"; pub const TMP_LOGS_DIR: &str = concatcp!(TMP_DIR, "/logs"); +pub const TMP_MEMORY_DIR: &str = concatcp!(TMP_DIR, "/memory"); pub const HUB_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "hub"); @@ -470,9 +512,8 @@ pub fn write_file(dir: &str, path: &str, content: &str) -> error::Result { Ok(file) } -pub fn write_file_bytes(dir: &str, path: &str, content: &Bytes) -> error::Result { - let path = format!("{}/{}", dir, path); - let mut file = File::create(&path)?; +pub fn write_file_bytes(path: &str, content: &Bytes) -> error::Result { + let mut file = File::create(path)?; file.write_all(content)?; file.flush()?; Ok(file) @@ -1034,6 +1075,7 @@ pub fn get_windmill_memory_usage() -> Option { } pub async fn update_min_version(conn: &Connection) -> bool { + tracing::debug!("Updating min version"); use crate::utils::{GIT_SEM_VERSION, GIT_VERSION}; let cur_version = GIT_SEM_VERSION.clone(); @@ -1065,6 +1107,9 @@ pub async fn update_min_version(conn: &Connection) -> bool { tracing::info!("Minimal worker version: {min_version}"); } + // Debouncing feature requires minimum version 1.566.0 across all workers + // This ensures all workers can handle debounce keys and stale data accumulation + *MIN_VERSION_SUPPORTS_DEBOUNCING.write().await = min_version >= Version::new(1, 566, 0); *MIN_VERSION_IS_AT_LEAST_1_461.write().await = min_version >= Version::new(1, 461, 0); *MIN_VERSION_IS_AT_LEAST_1_427.write().await = min_version >= Version::new(1, 427, 0); *MIN_VERSION_IS_AT_LEAST_1_432.write().await = min_version >= Version::new(1, 432, 0); @@ -1277,7 +1322,8 @@ pub async fn insert_ping_query( db: &DB, ) -> anyhow::Result<()> { sqlx::query!( - "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5", + "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (worker) + DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group", worker_instance, worker_name, ip, @@ -1352,7 +1398,11 @@ pub async fn update_job_ping_query( if let Some(i) = r { Ok(i) } else { - Err(anyhow::anyhow!("Job not found")) + Ok(PingJobStatusResponse { + canceled_by: None, + canceled_reason: None, + already_completed: true, + }) } } else { Err(to_anyhow(ro.unwrap_err())) diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 70be40e28b..edd9c8abe5 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -1,4 +1,6 @@ use async_recursion::async_recursion; +#[cfg(feature = "cloud")] +use backon::{ConstantBuilder, Retryable}; use quick_cache::sync::Cache; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -94,35 +96,56 @@ lazy_static::lazy_static! { } #[cfg(feature = "cloud")] -pub async fn get_team_plan_status(_db: &crate::DB, _w_id: &str) -> TeamPlanStatus { +pub async fn get_team_plan_status(_db: &crate::DB, _w_id: &str) -> Result { let cached = TEAM_PLAN_CACHE.get(_w_id); if let Some(cached) = cached { - return cached; + return Ok(cached); } - let team_plan_info = sqlx::query_as!( - TeamPlanStatus, - r#" - SELECT - w.premium, - COALESCE(cw.is_past_due, false) as "is_past_due!", - cw.max_tolerated_executions - FROM - workspace w - LEFT JOIN cloud_workspace_settings cw ON cw.workspace_id = w.id - WHERE - w.id = $1 - "#, - _w_id + + let team_plan_info = (|| async { + sqlx::query_as!( + TeamPlanStatus, + r#" + SELECT + w.premium, + COALESCE(cw.is_past_due, false) as "is_past_due!", + cw.max_tolerated_executions + FROM + workspace w + LEFT JOIN cloud_workspace_settings cw ON cw.workspace_id = w.id + WHERE + w.id = $1 + "#, + _w_id + ) + .fetch_optional(_db) + .await + }) + .retry( + ConstantBuilder::default() + .with_delay(std::time::Duration::from_secs(5)) + .with_max_times(10), ) - .fetch_one(_db) + .notify(|err, dur| { + tracing::error!( + "Failed to get team plan status for workspace {_w_id} (will retry in {dur:?}): {err:#}" + ); + }) .await - .unwrap_or_else(|_| TeamPlanStatus { + .map_err(|err| { + Error::internal_err(format!( + "Failed to get team plan status for workspace {_w_id} after 10 retries: {err:#}" + )) + })? + .unwrap_or_else(|| TeamPlanStatus { premium: false, is_past_due: false, max_tolerated_executions: None, }); + TEAM_PLAN_CACHE.insert(_w_id.to_string(), team_plan_info.clone()); - team_plan_info + + Ok(team_plan_info) } #[derive(Deserialize, Serialize, Debug)] @@ -212,7 +235,7 @@ pub async fn get_ducklake_from_db_unchecked( pub async fn get_ducklake_instance_pg_catalog_password(db: &DB) -> Result { sqlx::query_scalar!( - "SELECT trim(both '\"' from value::text) FROM global_settings WHERE name = 'ducklake_user_pg_pwd';" + "SELECT value->>'ducklake_user_pg_pwd' FROM global_settings WHERE name = 'ducklake_settings';" ) .fetch_optional(db) .await? @@ -268,7 +291,9 @@ async fn transform_json_unchecked( .await .map_err(to_anyhow)?; let mc = build_crypt(&db, &w_id).await?; - let variable = decrypt(&mc, variable)?; + let variable = decrypt(&mc, variable).map_err(|e| { + Error::internal_err(format!("Error decrypting variable {}: {}", &s, e)) + })?; serde_json::Value::String(variable) } s @ serde_json::Value::String(_) => s.clone(), diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.lock b/backend/windmill-duckdb-ffi-internal/Cargo.lock index 44353d9c7c..a53a988f30 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.lock +++ b/backend/windmill-duckdb-ffi-internal/Cargo.lock @@ -65,9 +65,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3f15b4c6b148206ff3a2b35002e08929c2462467b62b9c02036d9c34f9ef994" +checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" dependencies = [ "arrow-arith", "arrow-array", @@ -83,9 +83,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30feb679425110209ae35c3fbf82404a39a4c0436bb3ec36164d8bffed2a4ce4" +checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" dependencies = [ "arrow-array", "arrow-buffer", @@ -97,9 +97,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70732f04d285d49054a48b72c54f791bb3424abae92d27aafdf776c98af161c8" +checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" dependencies = [ "ahash 0.8.12", "arrow-buffer", @@ -107,15 +107,15 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown 0.15.5", + "hashbrown 0.16.0", "num", ] [[package]] name = "arrow-buffer" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "169b1d5d6cb390dd92ce582b06b23815c7953e9dfaaea75556e89d890d19993d" +checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" dependencies = [ "bytes", "half", @@ -124,9 +124,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4f12eccc3e1c05a766cafb31f6a60a46c2f8efec9b74c6e0648766d30686af8" +checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" dependencies = [ "arrow-array", "arrow-buffer", @@ -145,9 +145,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de1ce212d803199684b658fc4ba55fb2d7e87b213de5af415308d2fee3619c2" +checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" dependencies = [ "arrow-buffer", "arrow-schema", @@ -157,9 +157,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6506e3a059e3be23023f587f79c82ef0bcf6d293587e3272d20f2d30b969b5a7" +checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" dependencies = [ "arrow-array", "arrow-buffer", @@ -170,9 +170,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52bf7393166beaf79b4bed9bfdf19e97472af32ce5b6b48169d321518a08cae2" +checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" dependencies = [ "arrow-array", "arrow-buffer", @@ -183,18 +183,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7686986a3bf2254c9fb130c623cdcb2f8e1f15763e7c71c310f0834da3d292" +checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" dependencies = [ "bitflags", ] [[package]] name = "arrow-select" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd2b45757d6a2373faa3352d02ff5b54b098f5e21dccebc45a21806bc34501e5" +checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -206,9 +206,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0377d532850babb4d927a06294314b316e23311503ed580ec6ce6a0158f49d40" +checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" dependencies = [ "arrow-array", "arrow-buffer", @@ -363,11 +363,12 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.2.0" +version = "7.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f8e18d0dca9578507f13f9803add0df13362b02c501c1c17734f0dbb52eaf0b" +checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" dependencies = [ - "unicode-segmentation", + "strum 0.26.3", + "strum_macros 0.26.4", "unicode-width", ] @@ -414,9 +415,8 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "duckdb" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ab83a22530667ffc8cc0e31c0549bb07bea5dba3b957a8e315effc38923701" +version = "1.4.1" +source = "git+https://github.com/diegoimbert/duckdb-rs?branch=main#0df52a6941c9996d7ec60d585eaf6430db8c48cf" dependencies = [ "arrow", "cast", @@ -426,8 +426,7 @@ dependencies = [ "libduckdb-sys", "num-integer", "rust_decimal", - "smallvec", - "strum", + "strum 0.27.2", ] [[package]] @@ -550,6 +549,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" + [[package]] name = "hashlink" version = "0.10.0" @@ -697,9 +702,8 @@ checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "libduckdb-sys" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e02f6069513efb67a0743aff3b846090de14763802b0e95c352ebc6e1bdc1da" +version = "1.4.1" +source = "git+https://github.com/diegoimbert/duckdb-rs?branch=main#0df52a6941c9996d7ec60d585eaf6430db8c48cf" dependencies = [ "cc", "flate2", @@ -1106,25 +1110,38 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" -[[package]] -name = "smallvec" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" - [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" + [[package]] name = "strum" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros", + "strum_macros 0.27.2", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.106", ] [[package]] @@ -1225,12 +1242,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" - [[package]] name = "unicode-width" version = "0.2.1" diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.toml b/backend/windmill-duckdb-ffi-internal/Cargo.toml index ee3c58c1de..9560e1d0e6 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.toml +++ b/backend/windmill-duckdb-ffi-internal/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] chrono = "0.4.41" -duckdb = { version = "^1.3.2", features = ["bundled"] } +duckdb = { git = "https://github.com/diegoimbert/duckdb-rs", branch = "main", features = ["bundled"] } rust_decimal = "1.37.2" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "^1", features = ["preserve_order", "raw_value"] } diff --git a/backend/windmill-duckdb-ffi-internal/src/lib.rs b/backend/windmill-duckdb-ffi-internal/src/lib.rs index e2c918c3fe..29112baff9 100644 --- a/backend/windmill-duckdb-ffi-internal/src/lib.rs +++ b/backend/windmill-duckdb-ffi-internal/src/lib.rs @@ -1,11 +1,11 @@ use std::{ collections::HashMap, - ffi::{c_char, CStr, CString}, + ffi::{CStr, CString, c_char}, ptr::null_mut, }; -use duckdb::{params_from_iter, types::TimeUnit, Row}; -use rust_decimal::{prelude::FromPrimitive, Decimal}; +use duckdb::{Row, params_from_iter, types::TimeUnit}; +use rust_decimal::{Decimal, prelude::FromPrimitive}; use serde::Deserialize; use serde_json::value::RawValue; @@ -16,6 +16,17 @@ pub struct Arg { pub json_value: serde_json::Value, } +// Freeing from the caller side crashes the runtime with jemalloc enabled (EXIT CODE 11 SEGFAULT) +#[unsafe(no_mangle)] +pub extern "C" fn free_cstr(string: *mut c_char) -> () { + if string.is_null() { + return; + } + unsafe { + let _ = CString::from_raw(string); + } +} + #[unsafe(no_mangle)] pub extern "C" fn run_duckdb_ffi( query_block_list: *const *const c_char, @@ -217,15 +228,16 @@ fn do_duckdb_inner( if skip_collect { return Ok(RawValue::from_string("[]".to_string()).unwrap()); } - // Statement needs to be stepped at least once or stmt.column_names() will panic let mut column_names = None; + let mut type_aliases = None; loop { let row = rows.next(); match row { Ok(Some(row)) => { // Set column names if not already set let stmt = row.as_ref(); + let column_names = match column_names.as_ref() { Some(column_names) => column_names, None => { @@ -233,8 +245,20 @@ fn do_duckdb_inner( column_names.as_ref().unwrap() } }; + let type_aliases = match type_aliases.as_ref() { + Some(type_aliases) => type_aliases, + None => { + type_aliases = Some( + (0..stmt.column_count()) + .map(|i| stmt.column_logical_type(i).get_alias()) + .collect::>(), + ); + type_aliases.as_ref().unwrap() + } + }; - let row = row_to_value(row, &column_names.as_slice()).map_err(|e| e.to_string())?; + let row = row_to_value(row, &column_names.as_slice(), &type_aliases.as_slice()) + .map_err(|e| e.to_string())?; rows_vec.push(row); } Ok(None) => break, @@ -271,83 +295,99 @@ fn interpolate_named_args<'a>( (query, values) } -fn row_to_value(row: &Row<'_>, column_names: &[String]) -> Result, String> { +fn row_to_value( + row: &Row<'_>, + column_names: &[String], + type_aliases: &[Option], +) -> Result, String> { let mut obj = serde_json::Map::new(); for (i, key) in column_names.iter().enumerate() { let value: duckdb::types::Value = row.get(i).map_err(|e| e.to_string())?; - let json_value = match value { - duckdb::types::Value::Null => serde_json::Value::Null, - duckdb::types::Value::Boolean(b) => serde_json::Value::Bool(b), - duckdb::types::Value::TinyInt(i) => serde_json::Value::Number(i.into()), - duckdb::types::Value::SmallInt(i) => serde_json::Value::Number(i.into()), - duckdb::types::Value::Int(i) => serde_json::Value::Number(i.into()), - duckdb::types::Value::BigInt(i) => serde_json::Value::Number(i.into()), - duckdb::types::Value::HugeInt(i) => serde_json::Value::String(i.to_string()), - duckdb::types::Value::UTinyInt(u) => serde_json::Value::Number(u.into()), - duckdb::types::Value::USmallInt(u) => serde_json::Value::Number(u.into()), - duckdb::types::Value::UInt(u) => serde_json::Value::Number(u.into()), - duckdb::types::Value::UBigInt(u) => serde_json::Value::Number(u.into()), - duckdb::types::Value::Float(f) => serde_json::Value::Number( - serde_json::Number::from_f64(f as f64) - .ok_or_else(|| ("Could not convert to f64".to_string()))?, - ), - duckdb::types::Value::Double(f) => serde_json::Value::Number( - serde_json::Number::from_f64(f) - .ok_or_else(|| ("Could not convert to f64".to_string()))?, - ), - duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()), - duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()), - duckdb::types::Value::Text(s) => serde_json::Value::String(s), - duckdb::types::Value::Blob(b) => serde_json::Value::Array( - b.into_iter() - .map(|byte| serde_json::Value::Number(byte.into())) - .collect(), - ), - duckdb::types::Value::Date32(d) => serde_json::Value::Number(d.into()), - duckdb::types::Value::Time64(_, t) => serde_json::Value::String(t.to_string()), - duckdb::types::Value::Interval { months, days, nanos } => serde_json::json!({ - "months": months, - "days": days, - "nanos": nanos - }), - duckdb::types::Value::List(values) => serde_json::Value::Array( - values - .into_iter() - .map(|v| serde_json::Value::String(format!("{:?}", v))) - .collect(), - ), - duckdb::types::Value::Enum(e) => serde_json::Value::String(e), - duckdb::types::Value::Struct(fields) => serde_json::Value::Object( - fields - .iter() - .map(|(k, v)| (k.clone(), serde_json::Value::String(format!("{:?}", v)))) - .collect(), - ), - duckdb::types::Value::Array(values) => serde_json::Value::Array( - values - .into_iter() - .map(|v| serde_json::Value::String(format!("{:?}", v))) - .collect(), - ), - duckdb::types::Value::Map(map) => serde_json::Value::Object( - map.iter() - .map(|(k, v)| { - ( - format!("{:?}", k), - serde_json::Value::String(format!("{:?}", v)), - ) - }) - .collect(), - ), - duckdb::types::Value::Union(value) => { - serde_json::Value::String(format!("{:?}", *value)) - } - }; + let type_alias = &type_aliases[i]; + let json_value = duckdb_value_to_json_value(value, type_alias)?; obj.insert(key.clone(), json_value); } serde_json::value::to_raw_value(&obj).map_err(|e| e.to_string()) } +fn duckdb_value_to_json_value( + value: duckdb::types::Value, + type_alias: &Option, +) -> Result { + let json_value = match value { + duckdb::types::Value::Null => serde_json::Value::Null, + duckdb::types::Value::Boolean(b) => serde_json::Value::Bool(b), + duckdb::types::Value::TinyInt(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::SmallInt(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::Int(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::BigInt(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::HugeInt(i) => serde_json::Value::String(i.to_string()), + duckdb::types::Value::UTinyInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::USmallInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::UInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::UBigInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::Float(f) => serde_json::Value::Number( + serde_json::Number::from_f64(f as f64) + .ok_or_else(|| "Could not convert to f64".to_string())?, + ), + duckdb::types::Value::Double(f) => serde_json::Value::Number( + serde_json::Number::from_f64(f) + .ok_or_else(|| "Could not convert to f64".to_string())?, + ), + duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()), + duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()), + duckdb::types::Value::Text(s) if type_alias.as_deref().unwrap_or_default() == "JSON" => { + serde_json::from_str(&s) + .map_err(|e| format!("Error parsing JSON text: {}", e.to_string()))? + } + duckdb::types::Value::Text(s) => serde_json::Value::String(s), + duckdb::types::Value::Blob(b) => serde_json::Value::Array( + b.into_iter() + .map(|byte| serde_json::Value::Number(byte.into())) + .collect(), + ), + duckdb::types::Value::Date32(d) => serde_json::Value::Number(d.into()), + duckdb::types::Value::Time64(_, t) => serde_json::Value::String(t.to_string()), + duckdb::types::Value::Interval { months, days, nanos } => serde_json::json!({ + "months": months, + "days": days, + "nanos": nanos + }), + duckdb::types::Value::List(values) => serde_json::Value::Array( + values + .into_iter() + .map(|v| duckdb_value_to_json_value(v, &None)) + .collect::, _>>()?, + ), + duckdb::types::Value::Enum(e) => serde_json::Value::String(e), + duckdb::types::Value::Struct(fields) => serde_json::Value::Object( + fields + .iter() + .map(|(k, v)| duckdb_value_to_json_value(v.clone(), &None).map(|v| (k.clone(), v))) + .collect::, _>>()?, + ), + duckdb::types::Value::Array(values) => serde_json::Value::Array( + values + .into_iter() + .map(|v| duckdb_value_to_json_value(v, &None)) + .collect::, _>>()?, + ), + duckdb::types::Value::Map(map) => serde_json::Value::Object( + map.iter() + .map(|(k, v)| { + let k = match k { + duckdb::types::Value::Text(s) | duckdb::types::Value::Enum(s) => s.clone(), + _ => format!("{:?}", k), + }; + duckdb_value_to_json_value(v.clone(), &None).map(|v| (k, v)) + }) + .collect::, _>>()?, + ), + duckdb::types::Value::Union(value) => serde_json::Value::String(format!("{:?}", *value)), + }; + Ok(json_value) +} + fn json_value_to_duckdb_value( json_value: &serde_json::Value, arg_type: &str, @@ -401,7 +441,7 @@ fn json_value_to_duckdb_value( "double" | "float8" => duckdb::types::Value::Double(v), "decimal" | "numeric" => duckdb::types::Value::Decimal( Decimal::from_f64(v) - .ok_or_else(|| ("Could not convert f64 to Decimal".to_string()))?, + .ok_or_else(|| "Could not convert f64 to Decimal".to_string())?, ), _ => duckdb::types::Value::Double(v), // default fallback } @@ -436,7 +476,7 @@ fn string_to_duckdb_timestamp(s: &str) -> Result { fn string_to_duckdb_date(s: &str) -> Result { use chrono::Datelike; let date = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d") - .map_err(|e| (format!("Invalid date format: {}", e)))?; + .map_err(|e| format!("Invalid date format: {}", e))?; Ok(duckdb::types::Value::Date32(date.num_days_from_ce())) } diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index b50773f9c8..62b8b5962b 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -45,3 +45,4 @@ axum.workspace = true serde_urlencoded.workspace = true regex.workspace = true backon.workspace = true +quick_cache.workspace = true diff --git a/backend/windmill-queue/src/flow_status.rs b/backend/windmill-queue/src/flow_status.rs index 50af234149..e6b64c659d 100644 --- a/backend/windmill-queue/src/flow_status.rs +++ b/backend/windmill-queue/src/flow_status.rs @@ -1,6 +1,9 @@ use uuid::Uuid; use windmill_common::{ - error::{self, Error}, flows::Step, utils::WarnAfterExt, DB + error::{self, Error}, + flows::Step, + utils::WarnAfterExt, + DB, }; pub async fn update_flow_status_in_progress( @@ -11,7 +14,7 @@ pub async fn update_flow_status_in_progress( ) -> error::Result { let step = get_step_of_flow_status(db, flow).await?; match step { - Step::Step(step) => { + Step::Step { idx: step, .. } => { sqlx::query!( "UPDATE v2_job_status SET flow_status = jsonb_set( diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index fcefc458f7..d46a87ce0c 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -6,13 +6,15 @@ * LICENSE-AGPL for a copy of the license. */ +use std::future::Future; use std::{collections::HashMap, sync::Arc, vec}; use anyhow::Context; use async_recursion::async_recursion; -use chrono::{DateTime, Duration, Utc}; +use chrono::{DateTime, Utc}; use futures::future::TryFutureExt; use itertools::Itertools; +use quick_cache::sync::Cache; #[cfg(feature = "prometheus")] use prometheus::IntCounter; use regex::Regex; @@ -34,10 +36,9 @@ use windmill_common::auth::JobPerms; #[cfg(feature = "benchmark")] use windmill_common::bench::BenchmarkIter; use windmill_common::jobs::{JobTriggerKind, EMAIL_ERROR_HANDLER_USER_EMAIL}; -use windmill_common::utils::now_from_db; -use windmill_common::worker::{Connection, SCRIPT_TOKEN_EXPIRY}; -#[cfg(feature = "enterprise")] -use windmill_common::BASE_URL; +use windmill_common::utils::{configure_client, now_from_db}; +use windmill_common::worker::{Connection, MIN_VERSION_SUPPORTS_DEBOUNCING, SCRIPT_TOKEN_EXPIRY}; + use windmill_common::{ auth::{fetch_authed_from_permissioned_as, permissioned_as_to_username}, cache::{self, FlowData}, @@ -50,6 +51,7 @@ use windmill_common::{ }, flows::{ add_virtual_items_if_necessary, FlowModule, FlowModuleValue, FlowValue, InputTransform, + StopAfterIf, }, jobs::{get_payload_tag_from_prefixed_path, JobKind, JobPayload, QueuedJob, RawCode}, schedule::Schedule, @@ -67,7 +69,6 @@ use backon::ConstantBuilder; use backon::{BackoffBuilder, Retryable}; use crate::flow_status::{update_flow_status_in_progress, update_workflow_as_code_status}; -use crate::jobs_oss::update_concurrency_counter; use crate::schedule::{get_schedule_opt, push_scheduled_job}; use crate::tags::per_workspace_tag; #[cfg(feature = "cloud")] @@ -89,7 +90,7 @@ lazy_static::lazy_static! { ) .unwrap(); - static ref QUEUE_PULL_COUNT: prometheus::IntCounter = prometheus::register_int_counter!( + pub static ref QUEUE_PULL_COUNT: prometheus::IntCounter = prometheus::register_int_counter!( "queue_pull_count", "Total number of jobs pulled from the queue." ) @@ -100,10 +101,10 @@ lazy_static::lazy_static! { } lazy_static::lazy_static! { - pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new() + pub static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() .user_agent("windmill/beta") .timeout(std::time::Duration::from_secs(20)) - .connect_timeout(std::time::Duration::from_secs(10)) + .connect_timeout(std::time::Duration::from_secs(10))) .build().unwrap(); @@ -111,6 +112,9 @@ lazy_static::lazy_static! { .ok() .and_then(|x| x.parse().ok()) .unwrap_or(false); + + // TODO: Remove + static ref WMDEBUG_NO_DJOB_DEBOUNCING: bool = std::env::var("WMDEBUG_NO_DJOB_DEBOUNCING").is_ok(); } #[cfg(feature = "cloud")] @@ -121,13 +125,10 @@ const MAX_FREE_CONCURRENT_RUNS: i32 = 30; const ERROR_HANDLER_USERNAME: &str = "error_handler"; const SCHEDULE_ERROR_HANDLER_USERNAME: &str = "schedule_error_handler"; const GLOBAL_ERROR_HANDLER_USERNAME: &str = "global"; -#[cfg(feature = "enterprise")] -const SCHEDULE_RECOVERY_HANDLER_USERNAME: &str = "schedule_recovery_handler"; -const ERROR_HANDLER_USER_GROUP: &str = "g/error_handler"; -const ERROR_HANDLER_USER_EMAIL: &str = "error_handler@windmill.dev"; -const SCHEDULE_ERROR_HANDLER_USER_EMAIL: &str = "schedule_error_handler@windmill.dev"; -#[cfg(any(feature = "enterprise", feature = "cloud"))] -const SCHEDULE_RECOVERY_HANDLER_USER_EMAIL: &str = "schedule_recovery_handler@windmill.dev"; + +pub const ERROR_HANDLER_USER_GROUP: &str = "g/error_handler"; +pub const ERROR_HANDLER_USER_EMAIL: &str = "error_handler@windmill.dev"; +pub const SCHEDULE_ERROR_HANDLER_USER_EMAIL: &str = "schedule_error_handler@windmill.dev"; #[derive(Clone, Debug, Serialize, Deserialize)] pub struct CanceledBy { @@ -137,7 +138,7 @@ pub struct CanceledBy { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct JobCompleted { - pub job: Arc, + pub job: MiniCompletedJob, pub preprocessed_args: Option>>, pub result: Arc>, pub result_columns: Option>, @@ -147,23 +148,26 @@ pub struct JobCompleted { pub token: String, pub canceled_by: Option, pub duration: Option, + pub has_stream: Option, + pub from_cache: Option, } pub async fn cancel_single_job<'c>( username: &str, reason: Option, - job_running: Arc, + job_running: QueuedJobV2, w_id: &str, mut tx: Transaction<'c, Postgres>, db: &Pool, force_cancel: bool, ) -> error::Result<(Transaction<'c, Postgres>, Option)> { + + let id = job_running.id; if force_cancel || (job_running.parent_job.is_none() && !job_running.running) { let username = username.to_string(); let w_id = w_id.to_string(); let db = db.clone(); tracing::info!("cancelling job {:?}", job_running.id); - let job_running = job_running.clone(); tokio::task::spawn(async move { let reason: String = reason .clone() @@ -177,10 +181,11 @@ pub async fn cancel_single_job<'c>( &Connection::from(db.clone()), ) .await; + let memory_peak = job_running.memory_peak.unwrap_or(0); let add_job = add_completed_job_error( &db, - &MiniPulledJob::from(&job_running), - job_running.mem_peak.unwrap_or(0), + &MiniCompletedJob::from(job_running), + memory_peak, Some(CanceledBy { username: Some(username.to_string()), reason: Some(reason) }), e, "server", @@ -208,7 +213,7 @@ pub async fn cancel_single_job<'c>( } } - Ok((tx, Some(job_running.id))) + Ok((tx, Some(id))) } pub async fn cancel_job<'c>( @@ -221,26 +226,36 @@ pub async fn cancel_job<'c>( force_cancel: bool, require_anonymous: bool, ) -> error::Result<(Transaction<'c, Postgres>, Option)> { - let job = get_queued_job_tx(id, &w_id, &mut tx).await?; + //TODO fetch mini completed job instead of QueuedJob + let job = get_queued_job_v2(&mut *tx, &id).await?; + if job.is_none() { return Ok((tx, None)); } - if require_anonymous && job.as_ref().unwrap().created_by != "anonymous" { + let mut job = job.unwrap(); + + if require_anonymous && job.created_by != "anonymous" { return Err(Error::BadRequest( "You are not logged in and this job was not created by an anonymous user like you so you cannot cancel it".to_string(), )); } - let mut job = job.unwrap(); + + if job.workspace_id != w_id { + return Err(Error::BadRequest( + "You are not authorized to cancel this job belonging to another workspace".to_string(), + )); + } + if force_cancel { // if force canceling a flow step, make sure we force cancel from the highest parent loop { if job.parent_job.is_none() { break; } - match get_queued_job_tx(job.parent_job.unwrap(), &w_id, &mut tx).await? { + match get_queued_job_v2(&mut *tx, &job.parent_job.unwrap()).await? { Some(j) => { job = j; } @@ -250,7 +265,7 @@ pub async fn cancel_job<'c>( } // prevent cancelling a future tick of a schedule - if let Some(schedule_path) = job.schedule_path.as_ref() { + if let Some(schedule_path) = job.schedule_path().as_ref() { let now = now_from_db(&mut *tx).await?; if job.scheduled_for > now { return Err(Error::BadRequest( @@ -263,7 +278,7 @@ pub async fn cancel_job<'c>( } } - let job = Arc::new(job); + let job = job; // get all children using recursive CTE let mut jobs_to_cancel = sqlx::query!( @@ -305,7 +320,7 @@ ORDER BY depth, id let (ntx, _) = cancel_single_job( username, reason.clone(), - job.clone(), + job, w_id, tx, db, @@ -332,13 +347,13 @@ ORDER BY depth, id } } for job_id in jobs_to_cancel { - let job = get_queued_job_tx(job_id, &w_id, &mut tx).await?; + let job = get_queued_job_v2(&mut *tx, &job_id).await?; if let Some(job) = job { let (ntx, _) = cancel_single_job( username, reason.clone(), - Arc::new(job), + job, w_id, tx, db, @@ -375,7 +390,7 @@ pub async fn append_logs( match conn { Connection::Sql(pool) => { if let Err(err) = sqlx::query!( - "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, $1::text)", + "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, EXCLUDED.logs)", logs.as_ref(), job_id, workspace.as_ref(), @@ -426,6 +441,8 @@ pub async fn push_init_job<'c>( custom_concurrency_key: None, concurrent_limit: None, concurrency_time_window_s: None, + custom_debounce_key: None, + debounce_delay_s: None, cache_ttl: None, dedicated_worker: None, }), @@ -450,6 +467,8 @@ pub async fn push_init_job<'c>( None, None, false, + None, + None, ) .await?; inner_tx.commit().await?; @@ -480,6 +499,8 @@ pub async fn push_periodic_bash_job<'c>( custom_concurrency_key: None, concurrent_limit: None, concurrency_time_window_s: None, + custom_debounce_key: None, + debounce_delay_s: None, cache_ttl: None, dedicated_worker: None, }), @@ -504,6 +525,8 @@ pub async fn push_periodic_bash_job<'c>( None, None, false, + None, + None, ) .await?; inner_tx.commit().await?; @@ -543,7 +566,7 @@ async fn cancel_persistent_script_jobs_internal<'c>( // we could have retrieved the job IDs in the first query where we retrieve the hashes, but just in case a job was inserted in the queue right in-between the two above query, we re-do the fetch here let jobs_to_cancel = sqlx::query_scalar::<_, Uuid>( - "SELECT id FROM v2_as_queue WHERE workspace_id = $1 AND script_path = $2 AND canceled = false", + "SELECT j.id FROM v2_job_queue q JOIN v2_job j USING (id) WHERE j.workspace_id = $1 AND j.runnable_path = $2 AND q.canceled_by IS NULL", ) .bind(w_id) .bind(script_path) @@ -692,7 +715,7 @@ where pub async fn add_completed_job_error( db: &Pool, - queued_job: &MiniPulledJob, + completed_job: &MiniCompletedJob, mem_peak: i32, canceled_by: Option, e: serde_json::Value, @@ -703,7 +726,7 @@ pub async fn add_completed_job_error( #[cfg(feature = "prometheus")] register_metric( &WORKER_EXECUTION_FAILED, - &queued_job.tag, + &completed_job.tag, |s| { let counter = prometheus::register_int_counter!(prometheus::Opts::new( "worker_execution_failed", @@ -722,13 +745,13 @@ pub async fn add_completed_job_error( let result = WrappedError { error: e }; tracing::error!( "job {} in {} did not succeed: {}", - queued_job.id, - queued_job.workspace_id, + completed_job.id, + completed_job.workspace_id, serde_json::to_string(&result).unwrap_or_else(|_| "".to_string()) ); let _ = add_completed_job( db, - &queued_job, + &completed_job, false, false, Json(&result), @@ -737,6 +760,8 @@ pub async fn add_completed_job_error( canceled_by, flow_is_done, duration, + false, + false, ) .await?; Ok(result) @@ -745,11 +770,18 @@ pub async fn add_completed_job_error( lazy_static::lazy_static! { pub static ref GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE: Option = std::env::var("GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE").ok(); pub static ref MAX_RESULT_SIZE_MB: usize = std::env::var("MAX_RESULT_SIZE_MB").unwrap_or("500".to_string()).parse().unwrap_or(500); + + // Cache for restart_unless_cancelled flag - keyed by (hash, workspace_id) + static ref RESTART_UNLESS_CANCELLED_CACHE: Cache<(i64, String), bool> = Cache::new(10000); + + // Cache for workspace error handler settings with 60s TTL + // Key: workspace_id, Value: (error_handler, error_handler_extra_args, error_handler_muted_on_cancel, expiry_timestamp) + static ref WORKSPACE_ERROR_HANDLER_CACHE: Cache, Option>>, bool, i64)> = Cache::new(1000); } pub async fn add_completed_job( db: &Pool, - queued_job: &MiniPulledJob, + completed_job: &MiniCompletedJob, success: bool, skipped: bool, result: Json<&T>, @@ -758,7 +790,9 @@ pub async fn add_completed_job( canceled_by: Option, flow_is_done: bool, duration: Option, -) -> Result { + has_stream: bool, + from_cache: bool, +) -> Result<(Uuid, i64), Error> { // tracing::error!("Start"); // let start = tokio::time::Instant::now(); @@ -770,10 +804,10 @@ pub async fn add_completed_job( } let result_columns = result_columns.as_ref(); - let (opt_uuid, _duration, _skip_downstream_error_handlers) = (|| { + let (opt_uuid, duration, _skip_downstream_error_handlers) = (|| { commit_completed_job( db, - queued_job, + completed_job, success, skipped, result, @@ -782,6 +816,8 @@ pub async fn add_completed_job( &canceled_by, flow_is_done, duration, + has_stream, + from_cache, ) }) .retry( @@ -790,7 +826,11 @@ pub async fn add_completed_job( .with_max_times(5) .build(), ) - .when(|err| !matches!(err, Error::QuotaExceeded(_)) && !matches!(err, Error::ResultTooLarge(_))) + .when(|err| { + !matches!(err, Error::QuotaExceeded(_)) + && !matches!(err, Error::ResultTooLarge(_)) + && !matches!(err, Error::AlreadyCompleted(_)) + }) .notify(|err, dur| { tracing::error!("Could not insert completed job, retrying in {dur:#?}, err: {err:#?}"); }) @@ -799,16 +839,16 @@ pub async fn add_completed_job( // if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout if let Some(job_id) = opt_uuid { - return Ok(job_id); + return Ok((job_id, duration)); } #[cfg(feature = "cloud")] - apply_completed_job_cloud_usage(db, queued_job, _duration); + apply_completed_job_cloud_usage(db, completed_job, duration); - #[cfg(feature = "enterprise")] - apply_completed_job_error_handlers( + #[cfg(all(feature = "enterprise", feature = "private"))] + crate::jobs_ee::apply_completed_job_error_handlers( db, - queued_job, + completed_job, success, result, &canceled_by, @@ -816,16 +856,17 @@ pub async fn add_completed_job( ) .await; - restart_job_if_perpetual(db, queued_job, &canceled_by).await?; + restart_job_if_perpetual(db, completed_job, &canceled_by).await?; + // tracing::error!("4 {:?}", start.elapsed()); - Ok(queued_job.id) + Ok((completed_job.id, duration)) } async fn commit_completed_job( db: &Pool, - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, success: bool, skipped: bool, result: Json<&T>, @@ -834,6 +875,8 @@ async fn commit_completed_job( canceled_by: &Option, flow_is_done: bool, duration: Option, + has_stream: bool, + from_cache: bool, ) -> windmill_common::error::Result<(Option, i64, bool)> { // let start = std::time::Instant::now(); @@ -855,7 +898,7 @@ async fn commit_completed_job( return value; } - let _duration = sqlx::query_scalar!( + let duration = sqlx::query_scalar!( "INSERT INTO v2_job_completed AS cj ( workspace_id , id @@ -891,10 +934,33 @@ async fn commit_completed_job( /* $9 */ duration, /* $10 */ result_columns as Option<&Vec>, ) - .fetch_one(&mut *tx) + .fetch_optional(&mut *tx) .await .map_err(|e| Error::internal_err(format!("Could not add completed job {job_id}: {e:#}")))?; + let duration = if let Some(duration) = duration { + duration + } else { + let already_inserted = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM v2_job_completed WHERE id = $1)", + job_id + ) + .fetch_one(&mut *tx) + .await + .map_err(|e| Error::internal_err(format!("Could not add completed job {job_id}: {e:#}")))? + .unwrap_or(false); + + if already_inserted { + return Err(Error::AlreadyCompleted(format!( + "The queued job {job_id} is already completed." + ))); + } else { + return Err(Error::AlreadyCompleted(format!( + "There is no queued job anymore for {job_id} but there is no completed job either." + ))); + } + }; + if let Some(labels) = result.wm_labels() { sqlx::query!( "UPDATE v2_job SET labels = ( @@ -924,7 +990,7 @@ async fn commit_completed_job( ) WHERE id = $3", &queued_job.id.to_string(), - _duration, + duration, parent_job ) .execute(&mut *tx) @@ -991,9 +1057,11 @@ async fn commit_completed_job( } // for scripts, always try to schedule next tick - // for flows, only try to schedule next tick here if flow failed and because first handle_flow failed (step = 0, modules[0] = {type: 'Failure', 'job': uuid::nil()}) or job was cancelled before first handle_flow was called (step = 0, modules = [] OR modules[0].type == 'WaitingForPriorSteps') + // for flows, only try to schedule next tick here if flow failed and because first handle_flow failed (step = 0, modules[0] = {type: 'Failure', 'job': uuid::nil()}) + // or job was cancelled before first handle_flow was called (step = 0, modules = [] OR modules[0].type == 'WaitingForPriorSteps') // otherwise flow rescheduling is done inside handle_flow let schedule_next_tick = !queued_job.is_flow() + || from_cache || !success && sqlx::query_scalar!( "SELECT @@ -1017,13 +1085,13 @@ async fn commit_completed_job( .unwrap_or(false); if schedule_next_tick { - if let Err(err) = handle_maybe_scheduled_job( + if let Err(err) = Box::pin(handle_maybe_scheduled_job( db, queued_job, &schedule, &script_path, &queued_job.workspace_id, - ) + )) .await { match err { @@ -1034,8 +1102,8 @@ async fn commit_completed_job( }; } - #[cfg(feature = "enterprise")] - if let Err(err) = apply_schedule_handlers( + #[cfg(all(feature = "enterprise", feature = "private"))] + if let Err(err) = crate::jobs_ee::apply_schedule_handlers( db, &schedule, &script_path, @@ -1050,7 +1118,7 @@ async fn commit_completed_job( { if !success { tracing::error!("Could not apply schedule error handler: {}", err); - let base_url = BASE_URL.read().await; + let base_url = windmill_common::BASE_URL.read().await; let w_id: &String = &queued_job.workspace_id; if !matches!(err, Error::QuotaExceeded(_)) { report_error_to_workspace_handler_or_critical_side_channel( @@ -1116,15 +1184,21 @@ async fn commit_completed_job( .execute(&mut *tx) .await?; + if !success || has_stream { + sqlx::query!("DELETE FROM job_result_stream_v2 WHERE job_id = $1", job_id) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; tracing::info!( %job_id, root_job = ?queued_job.flow_innermost_root_job.map(|x| x.to_string()).unwrap_or_else(|| String::new()), - path = &queued_job.runnable_path(), + path = &queued_job.runnable_path, job_kind = ?queued_job.kind, started_at = ?queued_job.started_at.map(|x| x.to_string()).unwrap_or_else(|| String::new()), - duration = ?_duration, + duration = ?duration, permissioned_as = ?queued_job.permissioned_as, email = ?queued_job.permissioned_as_email, created_by = queued_job.created_by, @@ -1137,12 +1211,12 @@ async fn commit_completed_job( queued_job.id ); // tracing::info!("completed job: {:?}", start.elapsed().as_micros()); - Ok((None, _duration, _skip_downstream_error_handlers)) + Ok((None, duration, _skip_downstream_error_handlers)) } async fn check_result_size( db: &Pool, - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, result: Json<&T>, ) -> Option, i64, bool), Error>> { let result_size = result.size() / 1024 / 1024; @@ -1178,7 +1252,7 @@ async fn check_result_size( async fn restart_job_if_perpetual( db: &Pool, - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, canceled_by: &Option, ) -> Result<(), Error> { if !queued_job.is_flow_step() && queued_job.kind == JobKind::Script && canceled_by.is_none() { @@ -1204,18 +1278,27 @@ async fn restart_job_if_perpetual( async fn restart_job_if_perpetual_inner( db: &Pool, - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, hash: ScriptHash, ) -> Result<(), Error> { - let restart = sqlx::query_scalar!( - "SELECT restart_unless_cancelled FROM script WHERE hash = $1 AND workspace_id = $2", - hash.0, - &queued_job.workspace_id - ) - .fetch_optional(db) - .await? - .flatten() - .unwrap_or(false); + let cache_key = (hash.0, queued_job.workspace_id.clone()); + + let restart = if let Some(cached) = RESTART_UNLESS_CANCELLED_CACHE.get(&cache_key) { + cached + } else { + let restart = sqlx::query_scalar!( + "SELECT restart_unless_cancelled FROM script WHERE hash = $1 AND workspace_id = $2", + hash.0, + &queued_job.workspace_id + ) + .fetch_optional(db) + .await? + .flatten() + .unwrap_or(false); + + RESTART_UNLESS_CANCELLED_CACHE.insert(cache_key, restart); + restart + }; if restart { let tx = PushIsolationLevel::IsolatedRoot(db.clone()); @@ -1235,17 +1318,25 @@ async fn restart_job_if_perpetual_inner( None }; - let ehm = HashMap::new(); + let args = sqlx::query_scalar!( + "SELECT args as \"args: sqlx::types::Json>>\" FROM v2_job WHERE id = $1 AND workspace_id = $2", + queued_job.id, + queued_job.workspace_id + ) + .fetch_optional(db) + .await? + .flatten() + .unwrap_or_default(); let (_uuid, tx) = push( db, tx, &queued_job.workspace_id, JobPayload::ScriptHash { hash, - path: queued_job.runnable_path().to_string(), + path: queued_job.runnable_path.clone().unwrap_or_default(), custom_concurrency_key: custom_concurrency_key(db, &queued_job.id).await?, - concurrent_limit: queued_job.concurrent_limit, - concurrency_time_window_s: queued_job.concurrency_time_window_s, + concurrent_limit: None, + concurrency_time_window_s: None, cache_ttl: queued_job.cache_ttl, dedicated_worker: None, language: queued_job @@ -1254,12 +1345,11 @@ async fn restart_job_if_perpetual_inner( .unwrap_or_else(|| ScriptLang::Deno), priority: queued_job.priority, apply_preprocessor: false, + // TODO(debouncing): handle properly + custom_debounce_key: None, + debounce_delay_s: None, }, - queued_job - .args - .as_ref() - .map(|x| PushArgs::from(&x.0)) - .unwrap_or_else(|| PushArgs::from(&ehm)), + PushArgs::from(&args.0), &queued_job.created_by, &queued_job.permissioned_as_email, queued_job.permissioned_as.clone(), @@ -1273,13 +1363,15 @@ async fn restart_job_if_perpetual_inner( false, false, None, - queued_job.visible_to_owner, + true, Some(queued_job.tag.clone()), - queued_job.timeout, + None, None, queued_job.priority, None, false, + None, + None, ) .await?; tx.commit().await?; @@ -1287,131 +1379,10 @@ async fn restart_job_if_perpetual_inner( Ok(()) } -#[cfg(feature = "enterprise")] -async fn has_failure_module(db: &Pool, job: &MiniPulledJob) -> bool { - if let Ok(flow) = cache::job::fetch_flow(db, &job.kind, job.runnable_id).await { - return flow.value().failure_module.is_some(); - } - sqlx::query_scalar!( - "SELECT raw_flow->'failure_module' != 'null'::jsonb FROM v2_job WHERE id = $1", - job.id - ) - .fetch_one(db) - .await - .unwrap_or(Some(false)) - .unwrap_or(false) -} - -#[cfg(feature = "enterprise")] -async fn apply_completed_job_error_handlers( - db: &Pool, - queued_job: &MiniPulledJob, - success: bool, - result: Json<&T>, - canceled_by: &Option, - _skip_downstream_error_handlers: bool, -) { - if !success { - if queued_job.permissioned_as_email == ERROR_HANDLER_USER_EMAIL { - let base_url = BASE_URL.read().await; - let w_id = &queued_job.workspace_id; - report_critical_error( - format!( - "Workspace error handler job failed ({base_url}/run/{}?workspace={w_id}){}", - queued_job.id, - queued_job - .parent_job - .map(|id| format!( - " trying to handle failed job ({base_url}/run/{id}?workspace={w_id})" - )) - .unwrap_or("".to_string()), - ), - db.clone(), - Some(&w_id), - None, - ) - .await; - } else if queued_job.permissioned_as_email == SCHEDULE_ERROR_HANDLER_USER_EMAIL { - let base_url = BASE_URL.read().await; - let w_id = &queued_job.workspace_id; - report_error_to_workspace_handler_or_critical_side_channel( - &queued_job, - db, - format!( - "Schedule error handler job failed ({base_url}/run/{}?workspace={w_id}){}", - queued_job.id, - queued_job - .parent_job - .map(|id| format!( - " trying to handle failed job: {base_url}/run/{id}?workspace={w_id}" - )) - .unwrap_or("".to_string()), - ), - ) - .await; - } else if !_skip_downstream_error_handlers - && (matches!(queued_job.kind, JobKind::Script) - || matches!(queued_job.kind, JobKind::Flow) - && !has_failure_module(db, queued_job).await) - && queued_job.parent_job.is_none() - { - let result = serde_json::from_str( - &serde_json::to_string(result.0).unwrap_or_else(|_| "{}".to_string()), - ) - .unwrap_or_else(|_| json!({})); - let result = if result.is_object() || result.is_null() { - result - } else { - json!({ "error": result }) - }; - tracing::info!( - "Sending error of job {} to error handlers (if any)", - queued_job.id - ); - - if let Err(e) = send_error_to_global_handler(&queued_job, db, Json(&result)).await { - tracing::error!( - "Could not run global error handler for job {}: {}", - &queued_job.id, - e - ); - } - - if let Err(err) = send_error_to_workspace_handler( - &queued_job, - canceled_by.is_some(), - db, - Json(&result), - ) - .await - { - match err { - Error::QuotaExceeded(_) => {} - err => { - tracing::error!( - "Could not run workspace error handler for job {}: {}", - &queued_job.id, - err - ); - let base_url = BASE_URL.read().await; - let w_id: &String = &queued_job.workspace_id; - report_critical_error(format!( - "Failed to push workspace error handler job to handle failed job ({base_url}/run/{}?workspace={w_id}): {}", - queued_job.id, - err - ), db.clone(), Some(&w_id), None) - .await; - } - } - } - } - } -} - #[cfg(feature = "cloud")] fn apply_completed_job_cloud_usage( db: &Pool, - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, _duration: i64, ) { if *CLOUD_HOSTED && !queued_job.is_flow() && _duration > 1000 { @@ -1422,44 +1393,61 @@ fn apply_completed_job_cloud_usage( let email2 = email.clone(); tokio::task::spawn(async move { let additional_usage = _duration / 1000; - let premium_workspace = windmill_common::workspaces::get_team_plan_status(&db, &w_id) - .await - .premium; - tokio::time::timeout(std::time::Duration::from_secs(10), async move { - let _ = sqlx::query!( - "INSERT INTO usage (id, is_workspace, month_, usage) - VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) - ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", - w_id, + let result = tokio::time::timeout(std::time::Duration::from_secs(10), async move { + // Update workspace usage + let workspace_result = sqlx::query!( + "INSERT INTO usage (id, is_workspace, month_, usage) + VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) + ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + EXCLUDED.usage", + &w_id, additional_usage as i32 ) .execute(&db) - .await - .map_err(|e| { - Error::internal_err(format!("updating usage: {e:#}")) - }); + .await; - if !premium_workspace { - let _ = sqlx::query!( - "INSERT INTO usage (id, is_workspace, month_, usage) - VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) - ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", - email, - additional_usage as i32 - ) - .execute(&db) - .await - .map_err(|e| Error::internal_err(format!("updating usage: {e:#}"))); + if let Err(e) = workspace_result { + tracing::error!("Failed to update workspace usage for {}: {:#}", w_id, e); } - }).await.unwrap_or_else(|_| { - tracing::error!("Could not update usage for workspace {w_id2} and permissioned as {email2}, stopped after 10s"); - }); + + match windmill_common::workspaces::get_team_plan_status(&db, &w_id).await { + Ok(team_plan_status) => { + // Update user usage for non-premium workspaces + if !team_plan_status.premium { + let user_result = sqlx::query!( + "INSERT INTO usage (id, is_workspace, month_, usage) + VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) + ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + EXCLUDED.usage", + &email, + additional_usage as i32 + ) + .execute(&db) + .await; + + if let Err(e) = user_result { + tracing::error!("Failed to update user usage for {}: {:#}", email, e); + } + } + }, + Err(err) => { + tracing::error!("Failed to get team plan status to update usage for workspace {w_id}: {err:#}"); + } + }; + + }).await; + + if let Err(_) = result { + tracing::error!( + "Could not update usage for workspace {} and permissioned as {}, stopped after 10s", + w_id2, + email2 + ); + } }); } } pub async fn send_error_to_global_handler<'a, T: Serialize + Send + Sync>( - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, db: &Pool, result: Json<&T>, ) -> Result<(), Error> { @@ -1495,7 +1483,7 @@ pub async fn send_error_to_global_handler<'a, T: Serialize + Send + Sync>( } pub async fn report_error_to_workspace_handler_or_critical_side_channel( - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, db: &Pool, error_message: String, ) -> () { @@ -1557,32 +1545,76 @@ pub async fn report_error_to_workspace_handler_or_critical_side_channel( } pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>( - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, is_canceled: bool, db: &Pool, result: Json<&'a T>, ) -> Result<(), Error> { let w_id = &queued_job.workspace_id; - let row_result = sqlx::query_as::<_, (Option, Option>>, bool)>( - r#" - SELECT - error_handler, - error_handler_extra_args, - error_handler_muted_on_cancel - FROM - workspace_settings - WHERE - workspace_id = $1 - "#, - ) - .bind(&w_id) - .fetch_optional(db) - .await - .context("fetching error handler info from workspace_settings")? - .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))?; + // Try to get from cache first, checking if entry is still valid (within 60s TTL) + let now = chrono::Utc::now().timestamp(); + let (error_handler, error_handler_extra_args, error_handler_muted_on_cancel) = + if let Some(cached) = WORKSPACE_ERROR_HANDLER_CACHE.get(w_id) { + if cached.3 > now { + // Cache hit and not expired + (cached.0.clone(), cached.1.clone(), cached.2) + } else { + // Cache expired, fetch from database + let row_result = sqlx::query_as::<_, (Option, Option>>, bool)>( + r#" + SELECT + error_handler, + error_handler_extra_args, + error_handler_muted_on_cancel + FROM + workspace_settings + WHERE + workspace_id = $1 + "#, + ) + .bind(&w_id) + .fetch_optional(db) + .await + .context("fetching error handler info from workspace_settings")? + .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))?; - let (error_handler, error_handler_extra_args, error_handler_muted_on_cancel) = row_result; + // Update cache with 60s TTL + let expiry = now + 60; + WORKSPACE_ERROR_HANDLER_CACHE.insert( + w_id.clone(), + (row_result.0.clone(), row_result.1.clone(), row_result.2, expiry) + ); + row_result + } + } else { + // Cache miss, fetch from database + let row_result = sqlx::query_as::<_, (Option, Option>>, bool)>( + r#" + SELECT + error_handler, + error_handler_extra_args, + error_handler_muted_on_cancel + FROM + workspace_settings + WHERE + workspace_id = $1 + "#, + ) + .bind(&w_id) + .fetch_optional(db) + .await + .context("fetching error handler info from workspace_settings")? + .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))?; + + // Store in cache with 60s TTL + let expiry = now + 60; + WORKSPACE_ERROR_HANDLER_CACHE.insert( + w_id.clone(), + (row_result.0.clone(), row_result.1.clone(), row_result.2, expiry) + ); + row_result + }; if is_canceled && error_handler_muted_on_cancel { return Ok(()); @@ -1640,7 +1672,7 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> pub async fn handle_maybe_scheduled_job<'c>( db: &Pool, - job: &MiniPulledJob, + job: &MiniCompletedJob, schedule: &Schedule, script_path: &str, w_id: &str, @@ -1655,7 +1687,7 @@ pub async fn handle_maybe_scheduled_job<'c>( let push_next_job_future = (|| { tokio::time::timeout(std::time::Duration::from_secs(5), async { let mut tx = db.begin().await?; - tx = push_scheduled_job(db, tx, &schedule, None).await?; + tx = push_scheduled_job(db, tx, &schedule, None, Some(job.scheduled_for)).await?; tx.commit().await?; Ok::<(), Error>(()) }) @@ -1725,163 +1757,6 @@ pub async fn handle_maybe_scheduled_job<'c>( } } -#[cfg(feature = "enterprise")] -async fn apply_schedule_handlers<'a, 'c, T: Serialize + Send + Sync>( - db: &Pool, - schedule: &Schedule, - script_path: &str, - w_id: &str, - success: bool, - result: Json<&'a T>, - job_id: Uuid, - started_at: DateTime, - job_priority: Option, -) -> windmill_common::error::Result<()> { - if !success { - if let Some(on_failure_path) = schedule.on_failure.clone() { - let times = schedule.on_failure_times.unwrap_or(1).max(1); - let exact = schedule.on_failure_exact.unwrap_or(false); - if times > 1 || exact { - let past_jobs = sqlx::query!( - // Query plan: - // - use of the `ix_v2_job_root_by_path` index; - // hence the `parent_job IS NULL` clause. - // - select from `v2_job` first, then join with `v2_job_completed` to avoid a full - // table scan. - "SELECT status = 'success' AS \"success!\" - FROM v2_job j JOIN v2_job_completed USING (id) - WHERE j.workspace_id = $1 AND trigger_kind = 'schedule' AND trigger = $2 - AND parent_job IS NULL - AND runnable_path = $3 - AND j.id != $4 - ORDER BY created_at DESC - LIMIT $5", - &schedule.workspace_id, - &schedule.path, - script_path, - job_id, - if exact { times } else { times - 1 } as i64 - ) - .fetch_all(db) - .await?; - - let match_times = if exact { - past_jobs.len() == times as usize - && past_jobs[..(times - 1) as usize].iter().all(|j| !j.success) - && past_jobs[(times - 1) as usize].success - } else { - past_jobs.len() == ((times - 1) as usize) - && past_jobs.iter().all(|j| !j.success) - }; - - if !match_times { - return Ok(()); - } - } - - push_error_handler( - db, - job_id, - Some(schedule.path.to_string()), - Some(script_path.to_string()), - schedule.is_flow, - w_id, - &on_failure_path, - result, - Some(times), - Some(started_at), - schedule.on_failure_extra_args.clone(), - &schedule.email, - true, - false, - job_priority, - ) - .await?; - } - } else { - if let Some(ref on_success_path) = schedule.on_success { - handle_successful_schedule( - db, - job_id, - &schedule.path, - script_path, - schedule.is_flow, - w_id, - on_success_path, - result, - started_at, - schedule.on_success_extra_args.clone(), - ) - .await?; - } - - if let Some(ref on_recovery_path) = schedule.on_recovery.clone() { - let tx = db.begin().await?; - let times = schedule.on_recovery_times.unwrap_or(1).max(1); - let past_jobs = sqlx::query!( - // Query plan: - // - use of the `ix_v2_job_root_by_path` index; - // hence the `parent_job IS NULL` clause. - // - select from `v2_job` first, then join with `v2_job_completed` to avoid a full - // table scan. - "SELECT status = 'success' AS \"success!\", - result AS \"result: Json>\", - started_at AS \"started_at!\"\ - FROM v2_job j JOIN v2_job_completed USING (id) - WHERE j.workspace_id = $1 AND trigger_kind = 'schedule' AND trigger = $2 - AND parent_job IS NULL - AND runnable_path = $3 - AND j.id != $4 - ORDER BY created_at DESC - LIMIT $5", - &schedule.workspace_id, - &schedule.path, - script_path, - job_id, - times as i64 - ) - .fetch_all(db) - .await?; - - if past_jobs.len() < times as usize { - return Ok(()); - } - - let n_times_successful = past_jobs[..(times - 1) as usize].iter().all(|j| j.success); - - if !n_times_successful { - return Ok(()); - } - - let failed_job = &past_jobs[past_jobs.len() - 1]; - - if !failed_job.success { - handle_recovered_schedule( - db, - tx, - job_id, - &schedule.path, - script_path, - schedule.is_flow, - w_id, - &on_recovery_path, - failed_job.result.as_ref().map(AsRef::as_ref), - failed_job.started_at, - result, - times, - started_at, - schedule.on_recovery_extra_args.clone(), - ) - .await?; - } else { - tx.commit().await?; - } - } - } - - Ok(()) -} - pub const ERROR_HANDLER_PATH_TEAMS: &str = "/workspace-or-schedule-error-handler-teams"; pub const ERROR_HANDLER_PATH_SLACK: &str = "/workspace-or-schedule-error-handler-slack"; pub const ERROR_HANDLER_PATH_EMAIL: &str = "/workspace-or-error-handler-email"; @@ -2045,6 +1920,8 @@ pub async fn push_error_handler<'a, 'c, T: Serialize + Send + Sync>( priority, None, false, + None, + None, ) .await?; tx.commit().await?; @@ -2065,197 +1942,6 @@ fn sanitize_result(result: Json<&T>) -> HashMap( - db: &Pool, - tx: Transaction<'c, Postgres>, - job_id: Uuid, - schedule_path: &str, - script_path: &str, - is_flow: bool, - w_id: &str, - on_recovery_path: &str, - result: Option<&Box>, - started_at: DateTime, - successful_job_result: Json<&'a T>, - successful_times: i32, - successful_job_started_at: DateTime, - extra_args: Option>>, -) -> windmill_common::error::Result<()> { - let (payload, tag, on_behalf_of) = - get_payload_tag_from_prefixed_path(on_recovery_path, db, w_id).await?; - - let mut extra = HashMap::new(); - extra.insert("error_started_at".to_string(), to_raw_value(&started_at)); - extra.insert("schedule_path".to_string(), to_raw_value(&schedule_path)); - extra.insert("path".to_string(), to_raw_value(&script_path)); - extra.insert("is_flow".to_string(), to_raw_value(&is_flow)); - extra.insert( - "success_result".to_string(), - serde_json::from_str::>( - &serde_json::to_string(&successful_job_result).unwrap(), - ) - .unwrap_or_else(|_| serde_json::value::RawValue::from_string("{}".to_string()).unwrap()), - ); - extra.insert("success_times".to_string(), to_raw_value(&successful_times)); - extra.insert( - "success_started_at".to_string(), - to_raw_value(&successful_job_started_at), - ); - if let Some(args_v) = extra_args { - if let Ok(args_m) = serde_json::from_str::>>(args_v.get()) { - extra.extend(args_m); - } else { - return Err(error::Error::ExecutionErr( - "args of scripts needs to be dict".to_string(), - )); - } - } - - let args = result - .and_then(|x| serde_json::from_str::>>(x.get()).ok()) - .unwrap_or_else(HashMap::new); - - let (email, permissioned_as) = if let Some(on_behalf_of) = on_behalf_of.as_ref() { - ( - on_behalf_of.email.as_str(), - on_behalf_of.permissioned_as.clone(), - ) - } else { - ( - SCHEDULE_RECOVERY_HANDLER_USER_EMAIL, - ERROR_HANDLER_USER_GROUP.to_string(), - ) - }; - - let tx = PushIsolationLevel::Transaction(tx); - let (uuid, tx) = push( - &db, - tx, - w_id, - payload, - PushArgs { extra: Some(extra), args: &args }, - SCHEDULE_RECOVERY_HANDLER_USERNAME, - email, - permissioned_as, - Some(&format!("recovered.schedule.{job_id}")), - None, - None, - Some(job_id), - None, - Some(job_id), - None, - false, - false, - None, - true, - tag, - None, - None, - None, - None, - false, - ) - .await?; - tracing::info!( - "Pushed on_recovery job {} for {} to queue", - uuid, - schedule_path - ); - tx.commit().await?; - Ok(()) -} - -#[cfg(feature = "enterprise")] -async fn handle_successful_schedule<'a, 'c, T: Serialize + Send + Sync>( - db: &Pool, - job_id: Uuid, - schedule_path: &str, - script_path: &str, - is_flow: bool, - w_id: &str, - on_success_path: &str, - successful_job_result: Json<&'a T>, - successful_job_started_at: DateTime, - extra_args: Option>>, -) -> windmill_common::error::Result<()> { - let (payload, tag, on_behalf_of) = - get_payload_tag_from_prefixed_path(on_success_path, db, w_id).await?; - - let mut extra = HashMap::new(); - extra.insert("schedule_path".to_string(), to_raw_value(&schedule_path)); - extra.insert("path".to_string(), to_raw_value(&script_path)); - extra.insert("is_flow".to_string(), to_raw_value(&is_flow)); - extra.insert( - "success_result".to_string(), - serde_json::from_str::>( - &serde_json::to_string(&successful_job_result).unwrap(), - ) - .unwrap_or_else(|_| serde_json::value::RawValue::from_string("{}".to_string()).unwrap()), - ); - extra.insert( - "success_started_at".to_string(), - to_raw_value(&successful_job_started_at), - ); - if let Some(args_v) = extra_args { - if let Ok(args_m) = serde_json::from_str::>>(args_v.get()) { - extra.extend(args_m); - } else { - return Err(error::Error::ExecutionErr( - "args of scripts needs to be dict".to_string(), - )); - } - } - - let (email, permissioned_as) = if let Some(on_behalf_of) = on_behalf_of.as_ref() { - ( - on_behalf_of.email.as_str(), - on_behalf_of.permissioned_as.clone(), - ) - } else { - ( - SCHEDULE_RECOVERY_HANDLER_USER_EMAIL, - ERROR_HANDLER_USER_GROUP.to_string(), - ) - }; - - let tx = PushIsolationLevel::IsolatedRoot(db.clone()); - let (uuid, tx) = push( - &db, - tx, - w_id, - payload, - PushArgs { extra: Some(extra), args: &HashMap::new() }, - SCHEDULE_RECOVERY_HANDLER_USERNAME, - email, - permissioned_as, - Some(&format!("successful.schedule.recovery{job_id}")), - None, - None, - Some(job_id), - None, - Some(job_id), - None, - false, - false, - None, - true, - tag, - None, - None, - None, - None, - false, - ) - .await?; - tracing::info!( - "Pushed on_success job {} for {} to queue", - uuid, - schedule_path - ); - tx.commit().await?; - Ok(()) -} #[derive(sqlx::FromRow, Debug, Clone, Serialize, Deserialize)] pub struct MiniPulledJob { @@ -2291,6 +1977,148 @@ pub struct MiniPulledJob { pub trigger: Option, pub trigger_kind: Option, pub visible_to_owner: bool, + pub permissioned_as_end_user_email: Option, +} + + +#[derive(Debug, Clone, Serialize, Deserialize)] + +pub struct MiniCompletedJob { + pub id: Uuid, + pub workspace_id: String, + pub runnable_id: Option, + pub scheduled_for: chrono::DateTime, + pub parent_job: Option, + // pub root_job: Option, + pub flow_innermost_root_job: Option, + pub runnable_path: Option, + pub kind: JobKind, + pub started_at: Option>, + pub permissioned_as: String, + pub created_by: String, + pub script_lang: Option, + pub permissioned_as_email: String, + pub flow_step_id: Option, + pub trigger_kind: Option, + pub trigger: Option, + pub priority: Option, + pub concurrent_limit: Option, + pub tag: String, + pub cache_ttl: Option, +} + +impl From for MiniCompletedJob { + fn from(job: QueuedJobV2) -> Self { + MiniCompletedJob { + id: job.id, + workspace_id: job.workspace_id, + runnable_id: job.runnable_id, + scheduled_for: job.scheduled_for, + parent_job: job.parent_job, + flow_innermost_root_job: job.flow_innermost_root_job, + runnable_path: job.runnable_path, + kind: job.kind, + started_at: job.started_at, + permissioned_as: job.permissioned_as, + created_by: job.created_by, + script_lang: job.script_lang, + permissioned_as_email: job.permissioned_as_email, + flow_step_id: job.flow_step_id, + trigger_kind: job.trigger_kind, + trigger: job.trigger, + priority: job.priority, + concurrent_limit: job.concurrent_limit, + tag: job.tag, + cache_ttl: job.cache_ttl, + + } + } +} + +impl From for MiniCompletedJob { + fn from(job: MiniPulledJob) -> Self { + MiniCompletedJob { + id: job.id, + workspace_id: job.workspace_id, + runnable_id: job.runnable_id, + scheduled_for: job.scheduled_for, + parent_job: job.parent_job, + // root_job: job.root_job,, + flow_innermost_root_job: job.flow_innermost_root_job, + runnable_path: job.runnable_path, + kind: job.kind, + started_at: job.started_at, + permissioned_as: job.permissioned_as, + created_by: job.created_by, + script_lang: job.script_lang, + permissioned_as_email: job.permissioned_as_email, + flow_step_id: job.flow_step_id, + trigger_kind: job.trigger_kind, + trigger: job.trigger, + priority: job.priority, + concurrent_limit: job.concurrent_limit, + tag: job.tag, + cache_ttl: job.cache_ttl, + } + } +} + +impl From> for MiniCompletedJob { + fn from(job: Arc) -> Self { + MiniCompletedJob { + id: job.id, + workspace_id: job.workspace_id.clone(), + runnable_id: job.runnable_id, + scheduled_for: job.scheduled_for, + parent_job: job.parent_job, + flow_innermost_root_job: job.flow_innermost_root_job, + runnable_path: job.runnable_path.clone(), + kind: job.kind, + started_at: job.started_at, + permissioned_as: job.permissioned_as.clone(), + created_by: job.created_by.clone(), + script_lang: job.script_lang, + permissioned_as_email: job.permissioned_as_email.clone(), + flow_step_id: job.flow_step_id.clone(), + trigger_kind: job.trigger_kind.clone(), + trigger: job.trigger.clone(), + priority: job.priority, + concurrent_limit: job.concurrent_limit, + tag: job.tag.clone(), + cache_ttl: job.cache_ttl, + } + } +} + +impl MiniCompletedJob { + pub fn is_flow_step(&self) -> bool { + self.flow_step_id.is_some() + } + pub fn schedule_path(&self) -> Option { + schedule_path(&self.trigger_kind, &self.trigger) + } + + pub fn is_flow(&self) -> bool { + self.kind.is_flow() + } + + pub fn is_dependency(&self) -> bool { + self.kind.is_dependency() + } + +} + +fn schedule_path(trigger_kind: &Option, trigger: &Option) -> Option { + if trigger_kind.as_ref().is_some_and(|t| matches!(t, JobTriggerKind::Schedule)) { + trigger.clone() + } else { + None + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +struct FlowStatusChatInputEnabled { + chat_input_enabled: Option, } impl MiniPulledJob { @@ -2317,6 +2145,13 @@ impl MiniPulledJob { .and_then(|v| serde_json::from_str::((**v).get()).ok()) } + pub fn parse_chat_input_enabled(&self) -> Option { + self.flow_status + .as_ref() + .and_then(|v| serde_json::from_str::((**v).get()).ok()) + .and_then(|f| f.chat_input_enabled) + } + pub fn from(job: &QueuedJob) -> MiniPulledJob { MiniPulledJob { workspace_id: job.workspace_id.clone(), @@ -2355,6 +2190,7 @@ impl MiniPulledJob { None }, visible_to_owner: job.visible_to_owner.clone(), + permissioned_as_end_user_email: None, } } pub fn is_flow(&self) -> bool { @@ -2366,15 +2202,7 @@ impl MiniPulledJob { } pub fn schedule_path(&self) -> Option { - if self - .trigger_kind - .as_ref() - .is_some_and(|t| matches!(t, JobTriggerKind::Schedule)) - { - self.trigger.clone() - } else { - None - } + schedule_path(&self.trigger_kind, &self.trigger) } pub async fn mark_as_started_if_step(&self, db: &DB) -> Result<(), Error> { @@ -2513,8 +2341,14 @@ impl std::ops::Deref for PulledJob { } } +impl std::ops::DerefMut for PulledJob { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.job + } +} + lazy_static::lazy_static! { - static ref DISABLE_CONCURRENCY_LIMIT: bool = std::env::var("DISABLE_CONCURRENCY_LIMIT").is_ok_and(|s| s == "true"); + pub static ref DISABLE_CONCURRENCY_LIMIT: bool = std::env::var("DISABLE_CONCURRENCY_LIMIT").is_ok_and(|s| s == "true"); } pub async fn get_mini_pulled_job<'c>( @@ -2555,7 +2389,8 @@ pub async fn get_mini_pulled_job<'c>( script_entrypoint_override, trigger, trigger_kind as \"trigger_kind: JobTriggerKind\", - visible_to_owner + visible_to_owner, + NULL as permissioned_as_end_user_email FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id WHERE v2_job_queue.id = $1", job_id, ) @@ -2564,16 +2399,102 @@ pub async fn get_mini_pulled_job<'c>( Ok(job) } + +pub struct QueuedJobV2 { + pub id: Uuid, + pub workspace_id: String, + pub runnable_id: Option, + pub scheduled_for: chrono::DateTime, + pub parent_job: Option, + // pub root_job: Option, + pub flow_innermost_root_job: Option, + pub runnable_path: Option, + pub kind: JobKind, + pub started_at: Option>, + pub permissioned_as: String, + pub created_by: String, + pub script_lang: Option, + pub permissioned_as_email: String, + pub flow_step_id: Option, + pub trigger_kind: Option, + pub trigger: Option, + pub priority: Option, + pub concurrent_limit: Option, + pub tag: String, + pub cache_ttl: Option, + pub last_ping: Option>, + pub worker: Option, + pub memory_peak: Option, + pub running: bool, +} + +impl QueuedJobV2 { + pub fn schedule_path(&self) -> Option { + schedule_path(&self.trigger_kind, &self.trigger) + } +} + +pub async fn get_queued_job_v2<'c>( + e: impl PgExecutor<'c>, job_id: &Uuid) -> error::Result> { + let job = sqlx::query_as!( + QueuedJobV2, + "SELECT id, q.workspace_id, j.runnable_id as \"runnable_id: ScriptHash\", scheduled_for, parent_job, flow_innermost_root_job, runnable_path, kind as \"kind: JobKind\", started_at, permissioned_as, created_by, script_lang as \"script_lang: ScriptLang\", + permissioned_as_email, flow_step_id, trigger_kind as \"trigger_kind: JobTriggerKind\", trigger, q.priority, concurrent_limit, q.tag, cache_ttl, r.ping as last_ping, worker, memory_peak, running + FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id) + WHERE j.id = $1", + job_id, + + ) + .fetch_optional(e) + .await?; + Ok(job) +} + #[derive(Serialize, Deserialize, Debug)] pub struct PulledJobResult { pub job: Option, pub suspended: bool, + pub missing_concurrency_key: bool, } +pub enum PulledJobResultToJobErr { + MissingConcurrencyKey(JobCompleted), +} + +impl PulledJobResult { + pub fn to_pulled_job(self) -> Result, PulledJobResultToJobErr> { + match self { + PulledJobResult { job: Some(job), missing_concurrency_key: true, .. } => Err( + PulledJobResultToJobErr::MissingConcurrencyKey(JobCompleted { + preprocessed_args: None, + job: MiniCompletedJob::from(job.job), + success: false, + result: Arc::new(windmill_common::worker::to_raw_value(&json!({ + "name": "InternalErr", + "message": "The job has a concurrency limit but concurrency key couldn't be found. This is an unexpected behavior that should never happen. Please report this to support."} + ))), + result_columns: None, + mem_peak: 0, + cached_res_path: None, + token: "".to_string(), + canceled_by: None, + duration: None, + has_stream: Some(false), + from_cache: None, + }), + ), + PulledJobResult { job, .. } => Ok(job), + } + } +} + +/// Pull the job from queue pub async fn pull( db: &Pool, + // Whether or not try to pull from suspended jobs first suspend_first: bool, worker_name: &str, + // Execute queries supplied by caller instead of generic one query_o: Option<&(String, String)>, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> windmill_common::error::Result { @@ -2586,8 +2507,13 @@ pub async fn pull( } if pull_loop_count > 1000 { tracing::error!("Pull job loop count exceeded 1000, breaking"); - return Ok(PulledJobResult { job: None, suspended: false }); + return Ok(PulledJobResult { + job: None, + suspended: false, + missing_concurrency_key: false, + }); } + if let Some((query_suspended, query_no_suspend)) = query_o { let njob = { let job = if query_suspended.is_empty() { @@ -2598,41 +2524,70 @@ pub async fn pull( .fetch_optional(db) .await? }; - if let Some(job) = job { - PulledJobResult { job: Some(job), suspended: true } + + let (job, suspended) = if let Some(job) = job { + (Some(job), true) } else { let job = sqlx::query_as::<_, PulledJob>(query_no_suspend) .bind(worker_name) .fetch_optional(db) .await?; - PulledJobResult { job, suspended: false } + (job, false) + }; + + if let Some(job) = job.as_ref() { + if job.is_flow() || job.is_dependency() { + let per_workspace = per_workspace_tag(&job.workspace_id).await; + let base_tag = if job.is_flow() { + "flow".to_string() + } else { + "dependency".to_string() + }; + let tag = if per_workspace { + format!("{}-{}", base_tag, job.workspace_id) + } else { + base_tag + }; + sqlx::query!( + "UPDATE v2_job_queue SET tag = $1, running = false WHERE id = $2", + tag, + job.id + ) + .execute(db) + .await?; + continue; + } } - }; - if let Some(job) = njob.job.as_ref() { - if job.is_flow() || job.is_dependency() { - let per_workspace = per_workspace_tag(&job.workspace_id).await; - let base_tag = if job.is_flow() { - "flow".to_string() - } else { - "dependency".to_string() - }; - let tag = if per_workspace { - format!("{}-{}", base_tag, job.workspace_id) - } else { - base_tag - }; - sqlx::query!( - "UPDATE v2_job_queue SET tag = $1, running = false WHERE id = $2", - tag, - job.id - ) - .execute(db) - .await?; - continue; - } - } + + let pulled_job_result = match job { + #[cfg(feature = "private")] + Some(job) + if job.concurrent_limit.is_some() + // Concurrency limit is available for either enterprise job or dependency job + && (cfg!(feature = "enterprise") || (job.is_dependency() && !*WMDEBUG_NO_DJOB_DEBOUNCING)) => + { + let job = crate::jobs_ee::apply_concurrency_limit( + db, + pull_loop_count, + suspended, + job, + ) + .await?; + job.unwrap_or(PulledJobResult { + job: None, + suspended, + missing_concurrency_key: false, + }) + } + _ => PulledJobResult { job, suspended, missing_concurrency_key: false }, + }; + + Ok::<_, Error>(pulled_job_result) + }?; + return Ok(njob); }; + let (job, suspended) = pull_single_job_and_mark_as_running_no_concurrency_limit( db, suspend_first, @@ -2641,20 +2596,20 @@ pub async fn pull( bench, ) .await?; - let Some(job) = job else { - return Ok(PulledJobResult { job: None, suspended }); + return Ok(PulledJobResult { job: None, suspended, missing_concurrency_key: false }); }; let has_concurent_limit = job.concurrent_limit.is_some(); #[cfg(not(feature = "enterprise"))] - if has_concurent_limit { + if has_concurent_limit && !job.is_dependency() { tracing::error!("Concurrent limits are an EE feature only, ignoring constraints") } #[cfg(not(feature = "enterprise"))] - let has_concurent_limit = false; + let has_concurent_limit = job.is_dependency() && job.concurrent_limit.is_some() && cfg!(feature = "private") && !*WMDEBUG_NO_DJOB_DEBOUNCING; + // if we don't have private flag, we don't have concurrency limit // concurrency check. If more than X jobs for this path are already running, we re-queue and pull another job from the queue let pulled_job = job; @@ -2666,177 +2621,24 @@ pub async fn pull( if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { QUEUE_PULL_COUNT.inc(); } - return Ok(PulledJobResult { job: Some(pulled_job), suspended }); + return Ok(PulledJobResult { + job: Some(pulled_job), + suspended, + missing_concurrency_key: false, + }); } - let job_concurrency_key = concurrency_key(db, &pulled_job.id).await?; - if job_concurrency_key.is_none() { - tracing::warn!("No concurrency key found for job {}", pulled_job.id); - return Ok(PulledJobResult { job: None, suspended }); - } - let job_concurrency_key = job_concurrency_key.unwrap(); - tracing::debug!("Concurrency key is '{}'", job_concurrency_key); - let job_custom_concurrent_limit = pulled_job.concurrent_limit.unwrap(); - // setting concurrency_time_window to 0 will count only the currently running jobs - let job_custom_concurrency_time_window_s = - pulled_job.concurrency_time_window_s.unwrap_or(0); - tracing::debug!( - "Job concurrency limit is {} per {}s", - job_custom_concurrent_limit, - job_custom_concurrency_time_window_s - ); - - let jobs_uuids_init_json_value = serde_json::from_str::( - format!("{{\"{}\": {{}}}}", pulled_job.id.hyphenated().to_string()).as_str(), - ) - .expect("Unable to serialize job_uuids column to proper JSON"); - - let (within_limit, max_ended_at) = - if *DISABLE_CONCURRENCY_LIMIT || job_concurrency_key.is_empty() { - tracing::warn!("Concurrency limit is disabled, skipping"); - (true, None) - } else { - update_concurrency_counter( - db, - &pulled_job.id, - job_concurrency_key.clone(), - jobs_uuids_init_json_value, - pulled_job.id.hyphenated().to_string(), - job_custom_concurrency_time_window_s, - job_custom_concurrent_limit, - ) - .await? - }; - if within_limit { - #[cfg(feature = "prometheus")] - if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { - QUEUE_PULL_COUNT.inc(); - } - return Ok(PulledJobResult { job: Some(pulled_job), suspended }); - } - - let job_script_path = pulled_job.runnable_path.clone().unwrap_or_default(); - - let min_started_at = sqlx::query!( - "SELECT COALESCE((SELECT MIN(started_at) as min_started_at - FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id - WHERE v2_job.runnable_path = $1 AND v2_job.kind != 'dependencies' AND v2_job_queue.running = true AND v2_job_queue.workspace_id = $2 AND v2_job_queue.canceled_by IS NULL AND v2_job.concurrent_limit > 0), $3) as min_started_at, now() AS now", - job_script_path, - &pulled_job.workspace_id, - max_ended_at - ) - .fetch_one(db) - .await - .map_err(|e| { - Error::internal_err(format!( - "Error getting min started at for script path {job_script_path}: {e:#}" - )) - })?; - - let job_uuid: Uuid = pulled_job.id; - let avg_script_duration: Option = sqlx::query_scalar!( - "SELECT CAST(ROUND(AVG(duration_ms), 0) AS BIGINT) AS avg_duration_s FROM - (SELECT duration_ms FROM concurrency_key LEFT JOIN v2_job_completed ON v2_job_completed.id = concurrency_key.job_id WHERE key = $1 AND ended_at IS NOT NULL - ORDER BY ended_at - DESC LIMIT 10) AS t", - job_concurrency_key - ) - .fetch_one(db) - .await?; - tracing::debug!( - "avg script duration computed: {}", - avg_script_duration.unwrap_or(0) - ); - - // optimal scheduling is: 'older_job_in_concurrency_time_window_started_timestamp + script_avg_duration + concurrency_time_window_s' - let inc = Duration::try_milliseconds( - avg_script_duration.map(|x| i64::from(x + 100)).unwrap_or(0), - ) - .unwrap_or_default() - .max(Duration::try_seconds(1).unwrap_or_default()) - + Duration::try_seconds(i64::from(job_custom_concurrency_time_window_s)) - .unwrap_or_default(); - - let now = min_started_at.now.unwrap(); - let min_started_at_or_now = min_started_at.min_started_at.unwrap_or(now); - let min_started_p_inc = - (min_started_at_or_now + inc).max(now + Duration::try_seconds(3).unwrap_or_default()); - - let mut estimated_next_schedule_timestamp = min_started_p_inc; - let all_jobs = sqlx::query_scalar!( - "SELECT scheduled_for FROM v2_job_queue INNER JOIN concurrency_key ON concurrency_key.job_id = v2_job_queue.id - WHERE key = $1 AND running = false AND canceled_by IS NULL AND scheduled_for >= $2", - job_concurrency_key, - estimated_next_schedule_timestamp - inc - ).fetch_all(db).await?; - - tracing::debug!( - "all_jobs: {:?}, estimated_next_schedule_timestamp: {:?}, inc: {:?}", - all_jobs, - estimated_next_schedule_timestamp, - inc - ); - let mut i = 0; - loop { - let jobs_in_window = all_jobs - .iter() - .filter(|&scheduled_for| scheduled_for <= &estimated_next_schedule_timestamp) - .count() as i32 - - (job_custom_concurrent_limit * i); - - tracing::debug!("estimated_next_schedule_timestamp: {:?}, jobs_in_window: {jobs_in_window}, inc: {inc}", estimated_next_schedule_timestamp); - - if jobs_in_window < job_custom_concurrent_limit || *DISABLE_CONCURRENCY_LIMIT { - break; - } else { - i += 1; - estimated_next_schedule_timestamp = estimated_next_schedule_timestamp + inc; - } - if i % 50 == 0 { - tracing::warn!( - "Window finding for job {} loop count: {}", - job_uuid, - pull_loop_count - ); - tokio::task::yield_now().await; - } - if i > 1000000000 { - tracing::error!("Window finding job loop count exceeded 1000000000, breaking"); - break; + #[cfg(feature = "private")] + if cfg!(feature = "enterprise") + || (pulled_job.is_dependency() && !*WMDEBUG_NO_DJOB_DEBOUNCING) + { + if let Some(pulled_job) = + crate::jobs_ee::apply_concurrency_limit(db, pull_loop_count, suspended, pulled_job) + .await? + { + return Ok(pulled_job); } } - - tracing::info!("Job '{}' from path '{}' with concurrency key '{}' has reached its concurrency limit of {} jobs run in the last {} seconds. This job will be re-queued for next execution at {} (min_started_at: {min_started_at_or_now}, avg script duration: {:?}, number of time windows full: {})", - job_uuid, job_script_path, job_concurrency_key, job_custom_concurrent_limit, job_custom_concurrency_time_window_s, estimated_next_schedule_timestamp, avg_script_duration, i); - - let job_log_event = format!( - "\nRe-scheduled job to {estimated_next_schedule_timestamp} due to concurrency limits with key {job_concurrency_key} and limit {job_custom_concurrent_limit} in the last {job_custom_concurrency_time_window_s} seconds (min_started_at: {min_started_at_or_now}, avg script duration: {:?}, number of time windows full: {})\n", - avg_script_duration, i - ); - let _ = append_logs( - &job_uuid, - &pulled_job.workspace_id, - job_log_event, - &Connection::from(db.clone()), - ) - .await; - - sqlx::query!( - " - WITH ping AS ( - UPDATE v2_job_runtime SET ping = null WHERE id = $2 - ) - UPDATE v2_job_queue SET - running = false, - started_at = null, - scheduled_for = $1 - WHERE id = $2", - estimated_next_schedule_timestamp, - job_uuid, - ) - .execute(db) - .await - .map_err(|e| Error::internal_err(format!("Could not update and re-queue job {job_uuid}. The job will be marked as running but it is not running: {e:#}")))?; } } @@ -2870,6 +2672,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( } else { None }; + if r.is_none() { // #[cfg(feature = "benchmark")] // let instant = Instant::now(); @@ -2940,7 +2743,7 @@ pub async fn custom_concurrency_key( .await } -async fn concurrency_key( +pub async fn concurrency_key( db: &Pool, id: &Uuid, ) -> windmill_common::error::Result> { @@ -2957,6 +2760,69 @@ async fn concurrency_key( }) } +pub async fn custom_debounce_key( + db: &Pool, + job_id: &Uuid, +) -> Result, sqlx::Error> { + let fut = async || { + sqlx::query_scalar!("SELECT key FROM debounce_key WHERE job_id = $1", job_id) + .fetch_optional(db) // this should no longer be fetch optional + .await + }; + fut.retry( + ConstantBuilder::default() + .with_delay(std::time::Duration::from_secs(3)) + .with_max_times(5) + .build(), + ) + .notify(|err, dur| { + tracing::error!( + "Could not get debounce key for job {job_id}, retrying in {dur:#?}, err: {err:#?}" + ); + }) + .await +} + +/// Helper function to extract nodes/components to relock from job arguments +/// Returns the list of nodes to relock if present in either nodes_to_relock (flows) or components_to_relock (apps) +fn extract_to_relock_from_args(args: &HashMap>) -> Option> { + args.get("nodes_to_relock") // For flows + .or(args.get("components_to_relock")) // For apps + .and_then(|rv| { + serde_json::from_str::>(&rv.to_string()) + .map_err(|e| tracing::warn!("Failed to deserialize relock data: {}", e)) + .ok() + }) +} + +/// Helper function to accumulate nodes/components to relock for a debounced job +/// This merges new items with existing ones, removing duplicates +async fn accumulate_debounce_stale_data( + tx: &mut Transaction<'_, Postgres>, + job_id: &Uuid, + to_relock: &[String], +) -> Result<(), Error> { + sqlx::query!( + " + INSERT INTO debounce_stale_data (job_id, to_relock) + VALUES ($1, $2) + ON CONFLICT (job_id) + DO UPDATE SET to_relock = ( + SELECT array_agg(DISTINCT x) + FROM unnest( + -- Combine existing array with new values, removing duplicates + array_cat(debounce_stale_data.to_relock, EXCLUDED.to_relock) + ) AS x + ) + ", + job_id, + to_relock + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + pub fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> String { // Save this value to avoid parsing twice let workspaced = x.as_str().replace("$workspace", workspace_id).to_string(); @@ -3007,7 +2873,7 @@ pub fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> Strin } } -fn fullpath_with_workspace( +pub fn fullpath_with_workspace( workspace_id: &str, script_path: Option<&String>, job_kind: &JobKind, @@ -3438,33 +3304,29 @@ pub async fn job_is_complete(db: &DB, id: Uuid, w_id: &str) -> error::Result( - id: Uuid, - w_id: &str, - tx: &mut Transaction<'c, Postgres>, -) -> error::Result> { - sqlx::query_as::<_, QueuedJob>( - "SELECT *, null as workflow_as_code_status - FROM v2_as_queue WHERE id = $1 AND workspace_id = $2", - ) - .bind(id) - .bind(w_id) - .fetch_optional(&mut **tx) - .await - .map_err(Into::into) +pub fn get_mini_completed_job< +'a, +'e, +A: sqlx::Acquire<'e, Database = Postgres> + Send + 'a, +>(id: &'a Uuid, w_id: &'a str, db: A) -> impl Future>> + Send + 'a { + async move { + let mut conn = db.acquire().await?; + sqlx::query_as!( + MiniCompletedJob, + "SELECT + j.id, j.workspace_id, j.runnable_id AS \"runnable_id: ScriptHash\", q.scheduled_for, q.started_at, j.parent_job, j.flow_innermost_root_job, j.runnable_path, j.kind as \"kind!: JobKind\", j.permissioned_as, + j.created_by, j.script_lang AS \"script_lang: ScriptLang\", j.permissioned_as_email, j.flow_step_id, j.trigger_kind AS \"trigger_kind: JobTriggerKind\", j.trigger, j.priority, j.concurrent_limit, j.tag, j.cache_ttl + FROM v2_job j LEFT JOIN v2_job_queue q ON j.id = q.id + WHERE j.id = $1 AND j.workspace_id = $2", + id, + w_id + ) + .fetch_optional(&mut *conn) + .await + .map_err(Into::into) + } } -pub async fn get_queued_job(id: &Uuid, w_id: &str, db: &DB) -> error::Result> { - sqlx::query_as::<_, QueuedJob>( - "SELECT *, null as workflow_as_code_status - FROM v2_as_queue WHERE id = $1 AND workspace_id = $2", - ) - .bind(id) - .bind(w_id) - .fetch_optional(db) - .await - .map_err(Into::into) -} pub enum PushIsolationLevel<'c> { IsolatedRoot(DB), @@ -3508,7 +3370,7 @@ macro_rules! fetch_scalar_isolated { use sqlx::types::JsonRawValue; -#[derive(Debug, Default)] +#[derive(Debug, Clone, Default)] pub struct PushArgsOwned { pub extra: Option>>, pub args: HashMap>, @@ -3581,6 +3443,134 @@ lazy_static::lazy_static! { pub static ref RE_ARG_TAG: Regex = Regex::new(r#"\$args\[((?:\w+\.)*\w+)\]"#).unwrap(); } +#[cfg(feature = "cloud")] +lazy_static::lazy_static! { + // Cache for superadmin status: email -> (is_super_admin, expiry_timestamp) + static ref SUPERADMIN_CACHE: Arc>> = + Arc::new(RwLock::new(HashMap::new())); +} + +#[cfg(feature = "cloud")] +const SUPERADMIN_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(60); + +#[cfg(feature = "cloud")] +async fn is_superadmin_cached(db: &Pool, email: &str) -> Result { + let now = std::time::Instant::now(); + + // Try to get from cache first + { + let cache = SUPERADMIN_CACHE.read().await; + if let Some((is_super_admin, expiry)) = cache.get(email) { + if *expiry > now { + return Ok(*is_super_admin); + } + } + } + + // Cache miss or expired, fetch from database + let is_super_admin = + sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email) + .fetch_optional(db) + .await? + .unwrap_or(false); + + // Update cache + { + let mut cache = SUPERADMIN_CACHE.write().await; + cache.insert( + email.to_string(), + (is_super_admin, now + SUPERADMIN_CACHE_TTL), + ); + } + + Ok(is_super_admin) +} + +#[cfg(feature = "cloud")] +async fn check_usage_limits( + db: &Pool, + workspace_id: &str, + email: &str, + check_user_usage: bool, +) -> Result<(i32, Option), Error> { + // Get current workspace usage with a simple SELECT (no row lock) + let workspace_usage = sqlx::query_scalar!( + "SELECT usage FROM usage + WHERE id = $1 + AND is_workspace = TRUE + AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)", + workspace_id + ) + .fetch_optional(db) + .await + .map_err(|e| Error::internal_err(format!("fetching workspace usage: {e:#}")))? + .unwrap_or(0); + + // Get current user usage (only for non-premium workspaces) + let user_usage = if check_user_usage { + sqlx::query_scalar!( + "SELECT usage FROM usage + WHERE id = $1 + AND is_workspace = FALSE + AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)", + email + ) + .fetch_optional(db) + .await + .map_err(|e| Error::internal_err(format!("fetching user usage: {e:#}")))? + } else { + None + }; + + Ok((workspace_usage, user_usage)) +} + +#[cfg(feature = "cloud")] +fn increment_usage_async(db: Pool, workspace_id: String, email: Option) { + tokio::task::spawn(async move { + let result = tokio::time::timeout(std::time::Duration::from_secs(10), async { + // Update workspace usage + let workspace_result = sqlx::query!( + "INSERT INTO usage (id, is_workspace, month_, usage) + VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) + ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1", + &workspace_id + ) + .execute(&db) + .await; + + if let Err(e) = workspace_result { + tracing::error!("Failed to update workspace usage for {}: {:#}", workspace_id, e); + } + + // Update user usage if email is provided (non-premium workspaces only) + if let Some(ref email) = email { + let user_result = sqlx::query!( + "INSERT INTO usage (id, is_workspace, month_, usage) + VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) + ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1", + email + ) + .execute(&db) + .await; + + if let Err(e) = user_result { + tracing::error!("Failed to update user usage for {}: {:#}", email, e); + } + } + }) + .await; + + if let Err(_) = result { + tracing::error!( + "Usage update timed out after 10s for workspace {} and email {:?}", + workspace_id, + email + ); + } + }); +} + // #[instrument(level = "trace", skip_all)] pub async fn push<'c, 'd>( _db: &Pool, @@ -3592,7 +3582,8 @@ pub async fn push<'c, 'd>( mut email: &str, mut permissioned_as: String, token_prefix: Option<&str>, - scheduled_for_o: Option>, + #[allow(unused_mut)] + mut scheduled_for_o: Option>, schedule_path: Option, parent_job: Option, root_job: Option, @@ -3608,63 +3599,62 @@ pub async fn push<'c, 'd>( _priority_override: Option, authed: Option<&Authed>, running: bool, // whether the job is already running: only set this to true if you don't want the job to be picked up by a worker from the queue. It will also set started_at to now. + end_user_email: Option, + // If we know there is already a debounce job, we can use this for debouncing. + // NOTE: Only works with dependency jobs triggered by relative imports + debounce_job_id_o: Option, ) -> Result<(Uuid, Transaction<'c, Postgres>), Error> { #[cfg(feature = "cloud")] if *CLOUD_HOSTED { let team_plan_status = - windmill_common::workspaces::get_team_plan_status(_db, workspace_id).await; + windmill_common::workspaces::get_team_plan_status(_db, workspace_id).await?; // we track only non flow steps let (workspace_usage, user_usage) = if !matches!( job_payload, JobPayload::Flow { .. } | JobPayload::RawFlow { .. } ) { - tokio::time::timeout(std::time::Duration::from_secs(10), async move { - let workspace_usage = sqlx::query_scalar!( - "INSERT INTO usage (id, is_workspace, month_, usage) - VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) - ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 - RETURNING usage.usage", - workspace_id - ) - .fetch_one(_db) - .await - .map_err(|e| Error::internal_err(format!("updating usage: {e:#}")))?; + // Check current usage with SELECT (fast, no row locks) + // Only check user usage for non-premium workspaces + let (current_workspace_usage, current_user_usage) = + check_usage_limits(_db, workspace_id, email, !team_plan_status.premium).await?; - let user_usage = if !team_plan_status.premium { - Some(sqlx::query_scalar!( - "INSERT INTO usage (id, is_workspace, month_, usage) - VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) - ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 - RETURNING usage.usage", - email - ) - .fetch_one(_db) - .await - .map_err(|e| Error::internal_err(format!("updating usage: {e:#}")))?) + // Spawn async task to update usage counters in the background + increment_usage_async( + _db.clone(), + workspace_id.to_string(), + if !team_plan_status.premium { + Some(email.to_string()) } else { None - }; - Ok((Some(workspace_usage), user_usage)) - }).await.unwrap_or_else(|e| { - tracing::error!("Could not update usage for workspace {workspace_id} and permissioned as {email}, stopped after 10s: {e:#}"); - Err(Error::internal_err(format!("Could not update usage for workspace {workspace_id} and permissioned as {email}, stopped after 10s: {e:#}"))) - }) + }, + ); + + // Return the current usage + 1 to account for this job + let workspace_usage_with_new_job = current_workspace_usage + 1; + let user_usage_with_new_job = if !team_plan_status.premium { + Some(current_user_usage.unwrap_or(0) + 1) + } else { + None + }; + + (Some(workspace_usage_with_new_job), user_usage_with_new_job) } else { - Ok((None, None)) - }?; + (None, None) + }; if !team_plan_status.premium || team_plan_status.is_past_due { - let is_super_admin = - sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email) - .fetch_optional(_db) - .await? - .unwrap_or(false); + let is_super_admin = is_superadmin_cached(_db, email).await?; + + #[cfg(feature = "private")] + let recovery_email = crate::jobs_ee::SCHEDULE_RECOVERY_HANDLER_USER_EMAIL; + #[cfg(not(feature = "private"))] + let recovery_email = "recovery@windmill.dev"; if !is_super_admin { if !team_plan_status.premium && email != ERROR_HANDLER_USER_EMAIL && email != SCHEDULE_ERROR_HANDLER_USER_EMAIL - && email != SCHEDULE_RECOVERY_HANDLER_USER_EMAIL + && email != recovery_email && email != "worker@windmill.dev" && email != SUPERADMIN_SECRET_EMAIL && permissioned_as != SUPERADMIN_SYNC_EMAIL @@ -3697,7 +3687,7 @@ pub async fn push<'c, 'd>( } let in_queue = sqlx::query_scalar!( - "SELECT COUNT(id) FROM v2_as_queue WHERE email = $1", + "SELECT COUNT(id) FROM v2_job WHERE permissioned_as_email = $1", email ) .fetch_one(_db) @@ -3711,7 +3701,7 @@ pub async fn push<'c, 'd>( } let concurrent_runs = sqlx::query_scalar!( - "SELECT COUNT(id) FROM v2_as_queue WHERE running = true AND email = $1", + "SELECT COUNT(j.id) FROM v2_job_queue q JOIN v2_job j USING (id) WHERE q.running = true AND j.permissioned_as_email = $1", email ) .fetch_one(_db) @@ -3795,6 +3785,7 @@ pub async fn push<'c, 'd>( } let mut preprocessed = None; + #[allow(unused)] let ( script_hash, script_path, @@ -3803,12 +3794,14 @@ pub async fn push<'c, 'd>( raw_flow, flow_status, language, - custom_concurrency_key, - concurrent_limit, + mut custom_concurrency_key, + mut concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, _low_level_priority, + custom_debounce_key, + debounce_delay_s, ) = match job_payload { JobPayload::ScriptHash { hash, @@ -3821,6 +3814,8 @@ pub async fn push<'c, 'd>( dedicated_worker, priority, apply_preprocessor, + custom_debounce_key, + debounce_delay_s, } => { if apply_preprocessor { preprocessed = Some(false); @@ -3840,6 +3835,8 @@ pub async fn push<'c, 'd>( cache_ttl, dedicated_worker, priority, + custom_debounce_key, + debounce_delay_s, ) } JobPayload::FlowScript { @@ -3865,6 +3862,8 @@ pub async fn push<'c, 'd>( cache_ttl, dedicated_worker, None, + None, // custom_debounce_key removed for flow steps + None, // debounce_delay_s removed for flow steps ), JobPayload::FlowNode { id, path } => { let data = cache::flow::fetch_flow(_db, id).await?; @@ -3892,6 +3891,8 @@ pub async fn push<'c, 'd>( None, None, None, + None, + None, ) } JobPayload::AppScript { @@ -3913,6 +3914,8 @@ pub async fn push<'c, 'd>( cache_ttl, None, None, + None, + None, ), JobPayload::ScriptHub { path, apply_preprocessor } => { if path == "hub/7771/slack" || path == "hub/7836/slack" || path == "hub/9084/slack" { @@ -3945,6 +3948,8 @@ pub async fn push<'c, 'd>( None, None, None, + None, + None, ) } JobPayload::Code(RawCode { @@ -3958,6 +3963,8 @@ pub async fn push<'c, 'd>( concurrency_time_window_s, cache_ttl, dedicated_worker, + custom_debounce_key, + debounce_delay_s, }) => ( hash, path, @@ -3972,10 +3979,12 @@ pub async fn push<'c, 'd>( cache_ttl, dedicated_worker, None, + custom_debounce_key, + debounce_delay_s, ), JobPayload::Dependencies { hash, language, path, dedicated_worker } => ( Some(hash.0), - Some(path), + Some(path.clone()), None, JobKind::Dependencies, None, @@ -3987,7 +3996,11 @@ pub async fn push<'c, 'd>( None, dedicated_worker, None, + None, + None, ), + + // CLI usage, is not modifying db, no need for debouncing. JobPayload::RawScriptDependencies { script_path, content, language } => ( None, Some(script_path), @@ -4002,7 +4015,11 @@ pub async fn push<'c, 'd>( None, None, None, + None, + None, ), + + // CLI usage, is not modifying db, no need for debouncing. JobPayload::RawFlowDependencies { path, flow_value } => ( None, Some(path), @@ -4017,11 +4034,21 @@ pub async fn push<'c, 'd>( None, None, None, + None, + None, ), JobPayload::FlowDependencies { path, dedicated_worker, version } => { + #[cfg(test)] + let skip_compat = args + .args + .contains_key("dbg_create_job_for_unexistant_flow_version"); + + #[cfg(not(test))] + let skip_compat = false; + // Keep inserting `value` if not all workers are updated. // Starting at `v1.440`, the value is fetched on pull from the version id. - let value_o = if !*MIN_VERSION_IS_AT_LEAST_1_440.read().await { + let value_o = if !*MIN_VERSION_IS_AT_LEAST_1_440.read().await && !skip_compat { let mut ntx = tx.into_tx().await?; // The version has been inserted only within the transaction. let data = cache::flow::fetch_version(&mut *ntx, version).await?; @@ -4033,7 +4060,7 @@ pub async fn push<'c, 'd>( }; ( Some(version), - Some(path), + Some(path.clone()), None, JobKind::FlowDependencies, value_o, @@ -4045,11 +4072,13 @@ pub async fn push<'c, 'd>( None, dedicated_worker, None, + None, + None, ) } JobPayload::AppDependencies { path, version } => ( Some(version), - Some(path), + Some(path.clone()), None, JobKind::AppDependencies, None, @@ -4061,6 +4090,8 @@ pub async fn push<'c, 'd>( None, None, None, + None, + None, ), JobPayload::RawFlow { mut value, path, restarted_from } => { add_virtual_items_if_necessary(&mut value.modules); @@ -4098,6 +4129,9 @@ pub async fn push<'c, 'd>( }), user_states, preprocessor_module: None, + stream_job: None, + chat_input_enabled: None, + memory_id: None, } } _ => { @@ -4108,6 +4142,8 @@ pub async fn push<'c, 'd>( let concurrency_key = value.concurrency_key.clone(); let concurrent_limit = value.concurrent_limit; let concurrency_time_window_s = value.concurrency_time_window_s; + let debounce_key = value.debounce_key.clone(); + let debounce_delay_s = value.debounce_delay_s; let cache_ttl = value.cache_ttl.map(|x| x as i32); let priority = value.priority; ( @@ -4124,14 +4160,18 @@ pub async fn push<'c, 'd>( cache_ttl, None, priority, + debounce_key, + debounce_delay_s, ) } - JobPayload::SingleScriptFlow { + JobPayload::SingleStepFlow { path, hash, + flow_version, retry, error_handler_path, error_handler_args, + skip_handler, args, custom_concurrency_key, concurrent_limit, @@ -4141,11 +4181,79 @@ pub async fn push<'c, 'd>( tag_override, trigger_path, apply_preprocessor, + custom_debounce_key, + debounce_delay_s, } => { - let mut input_transforms = HashMap::::new(); - for (arg_name, arg_value) in args { - input_transforms.insert(arg_name, InputTransform::Static { value: arg_value }); + // Determine if this is a flow or a script + let is_flow = flow_version.is_some(); + + // Build modules list + let mut modules = vec![]; + + // Add skip validation module if provided + if let Some(skip_handler) = skip_handler { + let mut skip_input_transforms = HashMap::::new(); + for (arg_name, arg_value) in skip_handler.args { + skip_input_transforms + .insert(arg_name, InputTransform::Static { value: arg_value }); + } + + modules.push(FlowModule { + id: "skip_validation".to_string(), + value: to_raw_value(&FlowModuleValue::Script { + input_transforms: skip_input_transforms, + path: skip_handler.path, + hash: None, + tag_override: None, + is_trigger: None, + pass_flow_input_directly: None, + }), + stop_after_if: Some(StopAfterIf { + expr: skip_handler.stop_condition, + skip_if_stopped: true, + error_message: Some(skip_handler.stop_message), + }), + ..Default::default() + }); } + + // Add main module (script or flow) + let mut main_input_transforms = HashMap::::new(); + for (arg_name, arg_value) in args { + main_input_transforms.insert(arg_name, InputTransform::Static { value: arg_value }); + } + + let main_module = if is_flow { + FlowModule { + id: "a".to_string(), + value: to_raw_value(&FlowModuleValue::Flow { + path: path.clone(), + input_transforms: main_input_transforms, + pass_flow_input_directly: None, + }), + retry, + pass_flow_input_directly: Some(true), + ..Default::default() + } + } else { + FlowModule { + id: "a".to_string(), + value: to_raw_value(&FlowModuleValue::Script { + input_transforms: main_input_transforms, + path: path.clone(), + hash, + tag_override, + is_trigger: None, + pass_flow_input_directly: None, + }), + retry, + apply_preprocessor: Some(apply_preprocessor), + ..Default::default() + } + }; + modules.push(main_module); + + // Build failure module if error handler is provided let failure_module = if let Some(error_handler_path) = error_handler_path { let mut input_transforms = HashMap::::new(); input_transforms.insert( @@ -4158,7 +4266,7 @@ pub async fn push<'c, 'd>( ); input_transforms.insert( "is_flow".to_string(), - InputTransform::Static { value: to_raw_value(&false) }, + InputTransform::Static { value: to_raw_value(&is_flow) }, ); input_transforms.insert( "trigger_path".to_string(), @@ -4197,6 +4305,7 @@ pub async fn push<'c, 'd>( hash: None, tag_override: None, is_trigger: None, + pass_flow_input_directly: None, }), ..Default::default() })) @@ -4205,22 +4314,12 @@ pub async fn push<'c, 'd>( }; let flow_value = FlowValue { - modules: vec![FlowModule { - id: "a".to_string(), - value: to_raw_value(&FlowModuleValue::Script { - input_transforms, - path: path.clone(), - hash: Some(hash), - tag_override, - is_trigger: None, - }), - retry, - apply_preprocessor: Some(apply_preprocessor), - ..Default::default() - }], + modules, failure_module, concurrency_time_window_s, concurrent_limit, + debounce_key: custom_debounce_key.clone(), + debounce_delay_s, priority, cache_ttl: cache_ttl.map(|val| val as u32), concurrency_key: custom_concurrency_key.clone(), @@ -4228,14 +4327,15 @@ pub async fn push<'c, 'd>( early_return: None, skip_expr: None, preprocessor_module: None, + chat_input_enabled: None, }; // this is a new flow being pushed, flow_status is set to flow_value: let flow_status: FlowStatus = FlowStatus::new(&flow_value); ( - None, + None, // No version needed - flow is stored in raw_flow like FlowPreview Some(path), None, - JobKind::Flow, + JobKind::SingleStepFlow, Some(flow_value), Some(flow_status), None, @@ -4245,6 +4345,8 @@ pub async fn push<'c, 'd>( cache_ttl, None, priority, + custom_debounce_key, + debounce_delay_s, ) } JobPayload::Flow { path, dedicated_worker, apply_preprocessor, version } => { @@ -4272,11 +4374,16 @@ pub async fn push<'c, 'd>( let concurrency_time_window_s = value.concurrency_time_window_s; let mut concurrent_limit = value.concurrent_limit; + let custom_debounce_key = value.debounce_key.clone(); + let mut debounce_delay_s = value.debounce_delay_s; + if !apply_preprocessor { value.preprocessor_module = None; } else { tag = None; concurrent_limit = None; + // TODO: May be re-enable? + debounce_delay_s = None; preprocessed = Some(false); } @@ -4311,6 +4418,8 @@ pub async fn push<'c, 'd>( cache_ttl, dedicated_worker, priority, + custom_debounce_key, + debounce_delay_s, ) } JobPayload::RestartedFlow { completed_job_id, step_id, branch_or_iteration_n } => { @@ -4352,12 +4461,17 @@ pub async fn push<'c, 'd>( }), user_states, preprocessor_module: None, + stream_job: None, + chat_input_enabled: None, + memory_id: None, }; let value = flow_data.value(); let priority = value.priority; let concurrency_key = value.concurrency_key.clone(); let concurrent_limit = value.concurrent_limit; let concurrency_time_window_s = value.concurrency_time_window_s; + let debounce_key = value.debounce_key.clone(); + let debounce_delay_s = value.debounce_delay_s; let cache_ttl = value.cache_ttl.map(|x| x as i32); // Keep inserting `value` if not all workers are updated. // Starting at `v1.440`, the value is fetched on pull from the version id. @@ -4381,6 +4495,8 @@ pub async fn push<'c, 'd>( cache_ttl, None, priority, + debounce_key, + debounce_delay_s, ) } JobPayload::DeploymentCallback { path } => ( @@ -4397,6 +4513,8 @@ pub async fn push<'c, 'd>( None, None, None, + None, + None, ), JobPayload::Identity => ( None, @@ -4412,6 +4530,8 @@ pub async fn push<'c, 'd>( None, None, None, + None, + None, ), JobPayload::Noop => ( None, @@ -4427,6 +4547,8 @@ pub async fn push<'c, 'd>( None, None, None, + None, + None, ), JobPayload::AIAgent { path } => ( None, @@ -4442,9 +4564,27 @@ pub async fn push<'c, 'd>( None, None, None, + None, + None, ), }; + // Enforce concurrency limit on all dependency jobs. + // TODO: We can ignore this for scripts djobs. The main reason we need all djobs to be sequential is because we have + // nodes_to_relock and we need all locks whose corresponding steps aren't in nodes_to_relock be already present. + // + // This is not the case for scripts, so we can potentially have multiple djobs for scripts at the same time. + if let (Some(path), true) = ( + &script_path, + cfg!(feature = "private") + && job_kind.is_dependency() + && !*WMDEBUG_NO_DJOB_DEBOUNCING + && *MIN_VERSION_SUPPORTS_DEBOUNCING.read().await, + ) { + custom_concurrency_key = Some(format!("dependency:{workspace_id}/{path}")); + concurrent_limit = Some(1); + } + let final_priority: Option; #[cfg(not(feature = "enterprise"))] { @@ -4466,6 +4606,8 @@ pub async fn push<'c, 'd>( // prioritize flow steps to drain the queue faster let final_priority = if flow_step_id.is_some() && final_priority.is_none() { Some(0) + } else if job_kind == JobKind::Dependencies { + Some(0) } else { final_priority }; @@ -4575,6 +4717,211 @@ pub async fn push<'c, 'd>( Ulid::new().into() }; + // Dependency job debouncing: When multiple dependency jobs are scheduled for the same script/flow/app, + // we want to deduplicate them to avoid redundant work. The debouncing mechanism works by: + // 1. Creating a unique debounce key for each dependency target (dependency:workspace/type/path) + // 2. Reusing existing jobs when possible, or creating new ones when the existing job is already running + // 3. Accumulating the nodes/components that need relocking across all debounced requests + match ( + scheduled_for_o.is_some(), + job_kind.is_dependency(), + script_path.clone(), + *WMDEBUG_NO_DJOB_DEBOUNCING, + *MIN_VERSION_SUPPORTS_DEBOUNCING.read().await, + // We only do debouncing for jobs triggered by relative imports + // We do not want this be the case for normal djobs, since they will always be sequential. + args.args.contains_key("triggered_by_relative_import"), + ) { + (_, _, _, _, false, _) => { + tracing::warn!( + "Debouncing is disabled because workers are behind the minimum required version 1.566.0. \ + Please update workers to enable debouncing feature." + ); + } + // === DEPENDENCY JOB DEBOUNCING === + // + // Debouncing consolidates multiple dependency job requests into a single execution, + // reducing redundant work when many scripts/flows/apps are updated simultaneously. + // + // Prerequisites for debouncing (all must be true): + // 1. Job is scheduled in the future (debounce_delay is not None) - provides consolidation window + // 2. Job is a dependency job + // 3. Object path is provided (script/flow/app path) + // 4. Fallback mode is disabled (normal operation) + // 5. min version supports debouncing + // 6. Job was created by relative imports (triggered by dependency chain) + // + // How it works: + // + // PHASE 1 - PUSH (in jobs.rs::push): + // When a dependency job is scheduled with delay, check debounce_key table + // - If key exists: Merge request into existing job, accumulate nodes/components + // - If key doesn't exist: Create new entry and store initial nodes/components + // + // PHASE 2 - ACCUMULATION: + // During the debounce window (typically 5-15 seconds), multiple requests merge + // - Each request adds nodes/components to debounce_stale_data table + // - SQL DISTINCT automatically removes duplicates during merge + // + // PHASE 3 - PULL (in jobs.rs::pull): + // When the delayed job finally executes: + // - Lock debounce_key FOR UPDATE to prevent races + // - Retrieve all accumulated nodes/components from debounce_stale_data + // - Process all collected dependencies in single execution + // - Clean up both debounce_key and debounce_stale_data entries + (true, true, Some(obj_path), false, true, true) => { + // Generate unique debounce key: "workspace_id:object_path:dependency" + // This ensures each workspace+path combination has independent debounce window + let debounce_key = format!("{workspace_id}:{obj_path}:dependency"); + + tracing::debug!( + workspace_id = %workspace_id, + object_path = %obj_path, + debounce_key = %debounce_key, + "Checking for existing debounced dependency job" + ); + + // Check if there's already a pending job registered for this debounce key + // The debounce_job_id_o is passed in by the caller after locking the key FOR UPDATE + // IMPORTANT: This is assumed that the caller will lock debounce_key row in this transaction. + // We do this to block puller from further actions until we are done with consolidation and stuff that we do here in push. + if let Some(debounce_job_id) = debounce_job_id_o { + tracing::debug!( + existing_job_id = %debounce_job_id, + new_job_id = %job_id, + "Found existing debounced job, merging this request" + ); + + // NOTE: Race condition handling: + // In rare cases, the debounce_key entry may still exist even though the job + // has been pulled and is running. This can happen because: + // - Job pull marks job as running first + // - Then debounce_key cleanup happens (without transaction for performance) + // - Between these steps, new requests might see the old debounce_key + // + // This is acceptable because the puller will be blocked and cannot proceed until this transaction finishes. + // This will give us some space to add consolidated data (if such) and debounce the request. + // Once tx is commited, the puller will be unblocked and continue execution. + // Accumulate the nodes/components that need relocking from this request + + // This ensures all dependency updates are handled even if jobs are debounced + if let Some(to_relock) = extract_to_relock_from_args(&args.args) { + tracing::debug!( + job_id = %debounce_job_id, + node_count = to_relock.len(), + nodes = ?to_relock, + "Accumulating nodes/components to existing debounced job" + ); + + accumulate_debounce_stale_data(&mut tx, &debounce_job_id, &to_relock) + .await + .map_err(|e| { + tracing::error!( + error = %e, + job_id = %debounce_job_id, + debounce_key = %debounce_key, + "Failed to accumulate stale data for debounced job" + ); + e + })?; + } else { + tracing::trace!( + job_id = %debounce_job_id, + "No nodes to relock in this request, skipping accumulation" + ); + } + + // Return the existing job ID, effectively debouncing this request + // The new job_id we generated won't be used + tracing::debug!( + returned_job_id = %debounce_job_id, + skipped_job_id = %job_id, + "Debounced: returning existing job ID instead of creating new job" + ); + + // We will skip some of the work downstream and just debounce the job. + return Ok((debounce_job_id, tx)); + } else { + // No existing debounce entry - this is the first request in the debounce window + tracing::debug!( + job_id = %job_id, + debounce_key = %debounce_key, + "Creating new debounce entry (first request in window)" + ); + + sqlx::query!( + "INSERT INTO debounce_key (key, job_id) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET job_id = EXCLUDED.job_id", + &debounce_key, + job_id, + ) + .execute(&mut *tx) + .await + .map_err(|e| { + tracing::error!( + error = %e, + debounce_key = %debounce_key, + job_id = %job_id, + "Failed to insert debounce_key entry" + ); + Error::InternalErr(format!("Failed to create debounce entry: {}", e)) + })?; + + // Store initial nodes/components to relock if provided + if let Some(to_relock) = extract_to_relock_from_args(&args.args) { + tracing::debug!( + job_id = %job_id, + node_count = to_relock.len(), + nodes = ?to_relock, + "Storing initial nodes/components for new debounced job" + ); + + accumulate_debounce_stale_data(&mut tx, &job_id, &to_relock) + .await + .map_err(|e| { + tracing::error!( + error = %e, + job_id = %job_id, + "Failed to store initial stale data for debounced job" + ); + e + })?; + } else { + tracing::trace!( + job_id = %job_id, + "No initial nodes to relock, debounce entry created without stale data" + ); + } + } + } + _ => { + // Debouncing not applicable - proceed with normal job creation + tracing::trace!( + job_id = %job_id, + job_kind = ?job_kind, + "Debouncing conditions not met, proceeding with normal job creation" + ); + } + }; + + #[cfg(all(feature = "enterprise", feature = "private"))] + if schedule_path.is_none() { + if let Some(debounced_job_id) = crate::jobs_ee::maybe_apply_debouncing( + &job_id, + debounce_delay_s, + custom_debounce_key, + workspace_id, + script_path.clone(), + &job_kind, + &args, + &mut scheduled_for_o, + &mut tx, + ) + .await? + { + return Ok((debounced_job_id, tx)); + } + } + if concurrent_limit.is_some() { insert_concurrency_key( workspace_id, @@ -4587,7 +4934,6 @@ pub async fn push<'c, 'd>( ) .await?; } - let stringified_args = if *JOB_ARGS_AUDIT_LOGS { Some(serde_json::to_string(&args).map_err(|e| { Error::internal_err(format!( @@ -4682,9 +5028,9 @@ pub async fn push<'c, 'd>( INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null) ), inserted_job_perms AS ( - INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) - values ($1, $32, $33, $34, $35, $36, $37, $2) - ON CONFLICT (job_id) DO UPDATE SET email = $32, username = $33, is_admin = $34, is_operator = $35, folders = $36, groups = $37, workspace_id = $2 + INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, end_user_email) + values ($1, $32, $33, $34, $35, $36, $37, $2, $41) + ON CONFLICT (job_id) DO UPDATE SET email = EXCLUDED.email, username = EXCLUDED.username, is_admin = EXCLUDED.is_admin, is_operator = EXCLUDED.is_operator, folders = EXCLUDED.folders, groups = EXCLUDED.groups, workspace_id = EXCLUDED.workspace_id, end_user_email = EXCLUDED.end_user_email ) INSERT INTO v2_job_queue (workspace_id, id, running, scheduled_for, started_at, tag, priority) @@ -4733,6 +5079,7 @@ pub async fn push<'c, 'd>( root_job, trigger_kind as Option, running, + end_user_email, ) .execute(&mut *tx) .warn_after_seconds(1) @@ -4793,7 +5140,7 @@ pub async fn push<'c, 'd>( } JobKind::Flow => "jobs.run.flow", JobKind::FlowPreview => "jobs.run.flow_preview", - JobKind::SingleScriptFlow => "jobs.run.single_script_flow", + JobKind::SingleStepFlow => "jobs.run.single_step_flow", JobKind::Script_Hub => "jobs.run.script_hub", JobKind::Dependencies => "jobs.run.dependencies", JobKind::Identity => "jobs.run.identity", @@ -4876,6 +5223,39 @@ pub async fn insert_concurrency_key<'d, 'c>( Ok(()) } +// pub async fn insert_debounce_key<'d, 'c>( +// workspace_id: &str, +// args: &PushArgs<'d>, +// script_path: &Option, +// job_kind: JobKind, +// custom_concurrency_key: Option, +// tx: &mut Transaction<'c, Postgres>, +// job_id: Uuid, +// ) -> Result<(), Error> { +// let concurrency_key = custom_concurrency_key +// .map(|x| interpolate_args(x, args, workspace_id)) +// .unwrap_or(fullpath_with_workspace( +// workspace_id, +// script_path.as_ref(), +// &job_kind, +// )); +// sqlx::query!( +// "WITH inserted_concurrency_counter AS ( +// INSERT INTO concurrency_counter (concurrency_id, job_uuids) +// VALUES ($1, '{}'::jsonb) +// ON CONFLICT DO NOTHING +// ) +// INSERT INTO concurrency_key(key, job_id) VALUES ($1, $2)", +// concurrency_key, +// job_id, +// ) +// .execute(&mut **tx) +// .warn_after_seconds(3) +// .await +// .map_err(|e| Error::internal_err(format!("Could not insert concurrency_key={concurrency_key} for job_id={job_id} script_path={script_path:?} workspace_id={workspace_id}: {e:#}")))?; +// Ok(()) +// } + pub fn canceled_job_to_result(job: &MiniPulledJob) -> serde_json::Value { let reason = job .canceled_reason @@ -4905,11 +5285,11 @@ async fn restarted_flows_resolution( > { let row = sqlx::query!( "SELECT - script_path, script_hash AS \"script_hash: ScriptHash\", - job_kind AS \"job_kind!: JobKind\", - flow_status AS \"flow_status: Json>\", - raw_flow AS \"raw_flow: Json>\" - FROM v2_as_completed_job WHERE id = $1 and workspace_id = $2", + j.runnable_path as script_path, j.runnable_id AS \"script_hash: ScriptHash\", + j.kind AS \"job_kind!: JobKind\", + COALESCE(c.flow_status, c.workflow_as_code_status) AS \"flow_status: Json>\", + j.raw_flow AS \"raw_flow: Json>\" + FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.id = $1 and j.workspace_id = $2", completed_flow_id, workspace_id, ) @@ -4980,12 +5360,17 @@ async fn restarted_flows_resolution( if let Some(new_flow_jobs_success) = new_flow_jobs_success.as_mut() { new_flow_jobs_success.truncate(branch_or_iteration_n); } + let mut new_flow_jobs_timeline = module.flow_jobs_duration(); + if let Some(new_flow_jobs_timeline) = new_flow_jobs_timeline.as_mut() { + new_flow_jobs_timeline.truncate(branch_or_iteration_n); + } truncated_modules.push(FlowStatusModule::InProgress { id: module.id(), job: new_flow_jobs[new_flow_jobs.len() - 1], // set to last finished job from completed flow iterator: None, flow_jobs: Some(new_flow_jobs), flow_jobs_success: new_flow_jobs_success, + flow_jobs_duration: new_flow_jobs_timeline, branch_chosen: None, branchall: Some(BranchAllStatus { branch: branch_or_iteration_n - 1, // Doing minus one here as this variable reflects the latest finished job in the iteration @@ -5020,6 +5405,10 @@ async fn restarted_flows_resolution( if let Some(new_flow_jobs_success) = new_flow_jobs_success.as_mut() { new_flow_jobs_success.truncate(branch_or_iteration_n); } + let mut new_flow_jobs_timeline = module.flow_jobs_duration(); + if let Some(new_flow_jobs_timeline) = new_flow_jobs_timeline.as_mut() { + new_flow_jobs_timeline.truncate(branch_or_iteration_n); + } truncated_modules.push(FlowStatusModule::InProgress { id: module.id(), job: new_flow_jobs[new_flow_jobs.len() - 1], // set to last finished job from completed flow @@ -5029,6 +5418,7 @@ async fn restarted_flows_resolution( }), flow_jobs: Some(new_flow_jobs), flow_jobs_success: new_flow_jobs_success, + flow_jobs_duration: new_flow_jobs_timeline, branch_chosen: None, branchall: None, parallel, @@ -5136,7 +5526,7 @@ pub async fn get_same_worker_job( v2_job.raw_flow, pj.runnable_path as parent_runnable_path, p.email as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin, - p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders + p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id @@ -5155,3 +5545,240 @@ pub async fn get_same_worker_job( )) }) } + +pub async fn preprocess_dependency_job(job: &mut PulledJob, db: &DB) -> error::Result<()> { + let kind = job.kind; + // Handle dependency job debouncing cleanup when a job is pulled for execution + if kind.is_dependency() + && job + .args + .as_ref() + .map(|x| x.get("triggered_by_relative_import").is_some()) + .unwrap_or_default() + && !*WMDEBUG_NO_DJOB_DEBOUNCING + { + return Box::pin(async move { + // Only used for testing in tests/relative_imports.rs + // Give us some space to work with. + #[cfg(debug_assertions)] + if let Some(duration) = job + .args + .as_ref() + .map(|x| { + x.get("dbg_sleep_between_pull_and_debounce_key_removal") + .map(|v| serde_json::from_str::(v.get()).ok()) + .flatten() + }) + .flatten() + { + tracing::debug!("going to sleep",); + sleep(std::time::Duration::from_secs(duration as u64)).await; + } + + tracing::debug!( + "Processing debounce cleanup for dependency job {} at path {:?}", + &job.id, + &job.runnable_path + ); + + let key = format!("{}:{}:dependency", &job.workspace_id, job.runnable_path()); + let mut tx = db.begin().await?; + + // === DEBOUNCE CLEANUP === + // + // Clean up the debounce_key entry for this job (if it exists). + // + // IMPORTANT: We delete by key (not job_id) to avoid race conditions: + // If pusher has locked this row then this call will be blocked until all txs are commited. + // + // The idea is that the worker_lockfiles::trigger_dependents_to_recompute_locks will fetch the latest version of the obj. + // This object needs to be created before the djob is executed and it happens right here. + // + // This way the next pusher can fetch the latest version of object and base their djob payload on newest version. + // The concurrency limit on djobs will make sure that by the time next djob is started executing the base version it is referencing + // has already calculated all locks. This way even next djob will always use the fully finalized version of object. + // + // + // + // Note: We don't use a transaction here for performance (it's called during job pull). + // This means there's a tiny window where the job is running but key isn't deleted yet, + // which is acceptable because new requests will just accumulate data to this job. + tracing::debug!( + job_id = %job.id, + "Cleaning up debounce_key entry for completed/pulled job" + ); + + // This will either: + // 1. Block until pusher pushed. Which gives us: + // - If there was any stale data in pusher, then we will read it here (couple of lines below) + // 2. Block pusher until we are done here. This gives us: + // - We will clone objects and retrieve the latest version. So when we are done the pusher can read latest version. + sqlx::query!("DELETE FROM debounce_key WHERE key = $1", &key) + .execute(&mut *tx) + .await + .map_err(|e| { + tracing::error!( + error = %e, + job_id = %job.id, + "Failed to delete debounce_key" + ); + e + })?; + + let Some(base_hash) = job.runnable_id else { + return Err(Error::InternalErr( + "Missing runnable_id for dependency job triggered by relative import" + .to_string(), + )); + }; + + tracing::debug!( + job_id = %job.id, + base_hash = %base_hash, + job_kind = ?kind, + "Creating new version for dependency job triggered by relative import" + ); + + let new_id = match kind { + JobKind::Dependencies => { + let deployment_message = job + .args + .clone() + .map(|hashmap| { + hashmap + .get("deployment_message") + .map(|map_value| { + serde_json::from_str::(map_value.get()).ok() + }) + .flatten() + }) + .flatten(); + + // This way we tell downstream which script we should archive when the resolution is finished. + // (not used at the moment) + job.args.as_mut().map(|args| { + args.insert("base_hash".to_owned(), to_raw_value(&*base_hash)) + }); + + let new_hash = windmill_common::scripts::clone_script( + base_hash, + &job.workspace_id, + deployment_message, + &mut tx, + ) + .await?; + + new_hash + } + JobKind::FlowDependencies => { + sqlx::query_scalar!( + "INSERT INTO flow_version + (workspace_id, path, value, schema, created_by) + + SELECT workspace_id, path, value, schema, created_by + FROM flow_version WHERE path = $1 AND workspace_id = $2 AND id = $3 + + RETURNING id + ", + job.runnable_path(), + job.workspace_id, + *base_hash, + ) + .fetch_one(&mut *tx) + .await? + } + JobKind::AppDependencies => { + sqlx::query_scalar!( + "INSERT INTO app_version + (app_id, value, created_by, raw_app) + SELECT app_id, value, created_by, raw_app + FROM app_version WHERE id = $1 + RETURNING id", + *base_hash + ) + .fetch_one(&mut *tx) + .await? + } + _ => { + return Err(Error::InternalErr(format!( + "Matched unexpected JobKind ({:?}). This is a bug!", + kind + ))) + } + }; + + job.runnable_id.replace(new_id.into()); + + if !*windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING.read().await { + tx.commit().await?; + tracing::warn!("Debouncing is not supported on this version of Windmill. Minimum version required for debouncing support."); + return Ok(()); + } + // === RETRIEVE ACCUMULATED DEBOUNCE DATA === + // + // For flows and apps, retrieve all nodes/components that were accumulated + // during the debounce window. This data comes from requests that were merged + // into this job instead of creating their own jobs. + // + // Scripts don't need this because they don't have nodes/components to relock. + if let Some(to_relock_field) = match &job.kind { + JobKind::FlowDependencies => Some("nodes_to_relock"), + JobKind::AppDependencies => Some("components_to_relock"), + _ => None, // Scripts don't use accumulated stale data + } { + tracing::debug!( + job_id = %job.id, + job_kind = ?job.kind, + field = %to_relock_field, + "Retrieving accumulated stale data from debounced requests" + ); + + if let Some(stale_data) = sqlx::query_scalar!( + "DELETE FROM debounce_stale_data WHERE job_id = $1 RETURNING to_relock", + &job.id + ) + .fetch_optional(&mut *tx) + .await + .map_err(|e| { + tracing::error!( + error = %e, + job_id = %job.id, + "Failed to retrieve debounce_stale_data" + ); + e + })? + .flatten() + { + tracing::debug!( + job_id = %job.id, + node_count = stale_data.len(), + nodes = ?stale_data, + "Retrieved accumulated nodes/components from {} debounced requests", + stale_data.len() + ); + + // Replace the job's relock list with the accumulated data + // This ensures all nodes from all debounced requests are processed + if let Some(args) = job.args.as_mut() { + args.insert(to_relock_field.to_owned(), to_raw_value(&stale_data)); + tracing::debug!( + field = %to_relock_field, + "Updated job args with accumulated debounce data" + ); + } + } else { + tracing::trace!( + job_id = %job.id, + "No accumulated stale data found (no debounced requests or already cleaned up)" + ); + } + } + + // This will unblock pusher. + tx.commit().await?; + Ok(()) + }).await; + } + + Ok(()) +} diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 4fcd22771a..aed22e3f28 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -9,6 +9,8 @@ use crate::push; use crate::PushIsolationLevel; use anyhow::Context; +use chrono::DateTime; +use chrono::Utc; use sqlx::{PgExecutor, Postgres, Transaction}; use std::collections::HashMap; use std::str::FromStr; @@ -20,6 +22,9 @@ use windmill_common::get_latest_flow_version_info_for_path_from_version; use windmill_common::jobs::check_tag_available_for_workspace_internal; use windmill_common::jobs::JobPayload; use windmill_common::schedule::schedule_to_user; +use windmill_common::scripts::ScriptHash; +use windmill_common::utils::WarnAfterExt; +use windmill_common::worker::to_raw_value; use windmill_common::FlowVersionInfo; use windmill_common::DB; use windmill_common::{ @@ -29,11 +34,94 @@ use windmill_common::{ utils::{now_from_db, ScheduleType, StripPath}, }; +/// Helper to fetch metadata for a schedule's script or flow +async fn get_schedule_metadata<'c>( + tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, + schedule: &Schedule, +) -> Result<( + Option, // tag + Option, // timeout + Option, // on_behalf_of_email + String, // created_by + Option, // hash (for scripts) + Option, // flow_version (for flows) + Option, // retry +)> { + let parsed_retry = schedule + .retry + .clone() + .and_then(|r| serde_json::from_value::(r).ok()); + + if schedule.is_flow { + let version = get_latest_flow_version_id_for_path( + None, + &mut **tx, + &schedule.workspace_id, + &schedule.script_path, + false, + ) + .await?; + + let FlowVersionInfo { tag, on_behalf_of_email, edited_by, .. } = + get_latest_flow_version_info_for_path_from_version( + &mut **tx, + version, + &schedule.workspace_id, + &schedule.script_path, + ) + .await?; + + Ok(( + tag, + None, + on_behalf_of_email, + edited_by, + None, + Some(version), + parsed_retry, + )) + } else { + let ( + hash, + tag, + _custom_concurrency_key, + _concurrent_limit, + _concurrency_time_window_s, + _debounce_key, + _debounce_delay_s, + _cache_ttl, + _language, + _dedicated_worker, + _priority, + timeout, + on_behalf_of_email, + created_by, + ) = windmill_common::get_latest_hash_for_path( + &mut **tx, + &schedule.workspace_id, + &schedule.script_path, + false, + ) + .await?; + + Ok(( + tag, + timeout, + on_behalf_of_email, + created_by, + Some(hash), + None, + parsed_retry, + )) + } +} + pub async fn push_scheduled_job<'c>( db: &DB, mut tx: Transaction<'c, Postgres>, schedule: &Schedule, authed: Option<&Authed>, + now_cutoff: Option>, ) -> Result> { if !*LICENSE_KEY_VALID.read().await { return Err(error::Error::BadRequest( @@ -50,6 +138,19 @@ pub async fn push_scheduled_job<'c>( let now = now_from_db(&mut *tx).await?; + let now = match now_cutoff { + Some(now_cutoff) if now_cutoff >= now => { + tracing::error!( + "now_cutoff ({:?}) is after now ({:?}) for schedule {}. Using now_cutoff + 1s. This likely means the pg clock was shifted backwards.", + now_cutoff, + now, + &schedule.path + ); + now_cutoff + chrono::Duration::seconds(1) + } + _ => now, + }; + let starting_from = match schedule.paused_until { Some(paused_until) if paused_until > now => paused_until.with_timezone(&tz), paused_until_o => { @@ -60,6 +161,7 @@ pub async fn push_scheduled_job<'c>( &schedule.path ) .execute(&mut *tx) + .warn_after_seconds_with_sql(1, "update_schedule_paused_until".to_string()) .await .context("Failed to clear paused_until for schedule")?; } @@ -91,11 +193,12 @@ pub async fn push_scheduled_job<'c>( &schedule.script_path ) .fetch_one(&mut *tx) + .warn_after_seconds_with_sql(1, "already_exists_job".to_string()) .await? .unwrap_or(false); if already_exists { - tracing::info!( + tracing::warn!( "Job for schedule {} at {} already exists", &schedule.path, next @@ -117,7 +220,65 @@ pub async fn push_scheduled_job<'c>( } } - let (payload, tag, timeout, on_behalf_of_email, created_by) = if schedule.is_flow { + // If schedule handler is defined, wrap the scheduled job in a synthetic flow + // with the handler as the first step (with stop_after_if to skip if handler returns false) + let (payload, tag, timeout, on_behalf_of_email, created_by) = if let Some(handler_path) = + &schedule.dynamic_skip + { + // Build skip handler args + let mut skip_handler_args = HashMap::>::new(); + skip_handler_args.insert( + "scheduled_for".to_string(), + to_raw_value(&next.to_rfc3339()), + ); + + let stop_condition = "result !== true".to_string(); + let stop_message = format!( + "Schedule handler {} did not return true for datetime {}. Handler must return boolean true to execute scheduled job.", + handler_path, + next.to_rfc3339() + ); + + // Get metadata from the scheduled script/flow for tag, timeout, etc. + let (tag, timeout, on_behalf_of_email, created_by, hash, flow_version, retry) = + get_schedule_metadata(&mut tx, schedule).await?; + + ( + JobPayload::SingleStepFlow { + path: schedule.script_path.clone(), + hash, + flow_version, + args: args.clone(), + retry, + error_handler_path: None, + error_handler_args: None, + skip_handler: Some(windmill_common::jobs::SkipHandler { + path: handler_path.clone(), + args: skip_handler_args, + stop_condition, + stop_message, + }), + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + priority: None, + tag_override: schedule.tag.clone(), + trigger_path: None, + apply_preprocessor: false, + custom_debounce_key: None, + debounce_delay_s: None, + }, + if schedule.tag.as_ref().is_some_and(|x| x != "") { + schedule.tag.clone() + } else { + tag + }, + timeout, + on_behalf_of_email, + created_by, + ) + } else if schedule.is_flow { let version = get_latest_flow_version_id_for_path( None, &mut *tx, @@ -125,6 +286,7 @@ pub async fn push_scheduled_job<'c>( &schedule.script_path, false, ) + .warn_after_seconds_with_sql(1, "get_latest_flow_version_id_for_path".to_string()) .await?; let FlowVersionInfo { @@ -135,7 +297,12 @@ pub async fn push_scheduled_job<'c>( &schedule.workspace_id, &schedule.script_path, ) + .warn_after_seconds_with_sql( + 1, + "get_latest_flow_version_info_for_path_from_version".to_string(), + ) .await?; + ( JobPayload::Flow { path: schedule.script_path.clone(), @@ -155,6 +322,8 @@ pub async fn push_scheduled_job<'c>( custom_concurrency_key, concurrent_limit, concurrency_time_window_s, + custom_debounce_key, + debounce_delay_s, cache_ttl, language, dedicated_worker, @@ -168,6 +337,7 @@ pub async fn push_scheduled_job<'c>( &schedule.script_path, false, ) + .warn_after_seconds_with_sql(1, "get_latest_hash_for_path".to_string()) .await?; if schedule.retry.is_some() { @@ -184,21 +354,25 @@ pub async fn push_scheduled_job<'c>( } // if retry is set, we wrap the script into a one step flow with a retry on the module ( - JobPayload::SingleScriptFlow { + JobPayload::SingleStepFlow { path: schedule.script_path.clone(), - hash: hash, + hash: Some(hash), + flow_version: None, retry: Some(parsed_retry), error_handler_path: None, error_handler_args: None, + skip_handler: None, args: static_args, custom_concurrency_key: None, concurrent_limit: None, concurrency_time_window_s: None, - cache_ttl: cache_ttl, - priority: priority, + cache_ttl, + priority, tag_override: schedule.tag.clone(), trigger_path: None, apply_preprocessor: false, + custom_debounce_key: None, + debounce_delay_s: None, }, if schedule.tag.as_ref().is_some_and(|x| x != "") { schedule.tag.clone() @@ -215,13 +389,15 @@ pub async fn push_scheduled_job<'c>( hash, path: schedule.script_path.clone(), custom_concurrency_key, - concurrent_limit: concurrent_limit, - concurrency_time_window_s: concurrency_time_window_s, - cache_ttl: cache_ttl, + concurrent_limit, + concurrency_time_window_s, + cache_ttl, dedicated_worker, language, priority, apply_preprocessor: false, + custom_debounce_key, + debounce_delay_s, }, if schedule.tag.as_ref().is_some_and(|x| x != "") { schedule.tag.clone() @@ -241,6 +417,7 @@ pub async fn push_scheduled_job<'c>( &schedule.path ) .execute(&mut *tx) + .warn_after_seconds_with_sql(1, "clear_schedule_error".to_string()) .await { tracing::error!( @@ -256,10 +433,12 @@ pub async fn push_scheduled_job<'c>( let is_windmill_user = sqlx::query_scalar!("SELECT CURRENT_USER = 'windmill_user' as \"is_windmill_user!\"") .fetch_one(&mut *tx) + .warn_after_seconds_with_sql(1, "is_windmill_user".to_string()) .await?; if is_windmill_user { sqlx::query!("SET LOCAL ROLE NONE") .execute(&mut *tx) + .warn_after_seconds_with_sql(1, "set_local_role_none".to_string()) .await?; } ( @@ -285,9 +464,16 @@ pub async fn push_scheduled_job<'c>( email, None, // no token for schedules so no scopes so no scope_tags ) + .warn_after_seconds_with_sql(1, "check_tag_available_for_workspace_internal".to_string()) .await?; } + tracing::info!( + "Pushing next scheduled job for schedule {} at {} (schedule: {})", + &schedule.path, + next, + &schedule.schedule + ); let tx = PushIsolationLevel::Transaction(tx); let (_, mut tx) = push( &db, @@ -315,12 +501,16 @@ pub async fn push_scheduled_job<'c>( None, push_authed, false, + None, + None, ) + .warn_after_seconds_with_sql(1, "push in push_scheduled_job".to_string()) .await?; if revert_to_windmill_user { sqlx::query!("SET LOCAL ROLE windmill_user") .execute(&mut *tx) + .warn_after_seconds_with_sql(1, "set_local_role_windmill_user".to_string()) .await?; } diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 772a6195f9..f36d2c5d58 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -22,6 +22,7 @@ cloud = [] sqlx = [] deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core", "dep:deno_ast", "dep:deno_tls", "dep:deno_permissions", "dep:deno_io", "dep:deno_runtime", "dep:deno_telemetry", "dep:deno_error", "dep:winapi"] +libffi_mac = ["dep:libffi-sys"] otel = ["windmill-common/otel", "dep:opentelemetry"] dind = ["dep:bollard"] php = ["dep:windmill-parser-php"] @@ -56,6 +57,7 @@ windmill-parser-sql.workspace = true windmill-parser-graphql.workspace = true windmill-parser-php = { workspace = true, optional = true } windmill-git-sync.workspace = true +rmcp = { version = "0.8.1", features = ["client", "transport-streamable-http-client", "transport-streamable-http-client-reqwest"] } flume.workspace = true sqlx.workspace = true uuid.workspace = true @@ -67,6 +69,7 @@ serde.workspace = true serde_json.workspace = true futures.workspace = true async-recursion.workspace = true +async-trait.workspace = true anyhow.workspace = true itertools.workspace = true regex.workspace = true @@ -112,6 +115,7 @@ nix.workspace = true bytes.workspace = true reqwest.workspace = true reqwest-middleware.workspace = true +mime_guess.workspace = true hex.workspace = true tiberius = { workspace = true, optional = true } tokio-util = { workspace = true, optional = true } @@ -145,3 +149,4 @@ deno_io = { workspace = true, optional = true } deno_runtime = { workspace = true, optional = true } deno_telemetry = { workspace = true, optional = true } winapi = { workspace = true, optional = true } +libffi-sys = { workspace = true, optional = true } diff --git a/backend/windmill-worker/loader.bun.js b/backend/windmill-worker/loader.bun.js index 857bc95573..289fac0a9a 100644 --- a/backend/windmill-worker/loader.bun.js +++ b/backend/windmill-worker/loader.bun.js @@ -15,8 +15,10 @@ const p = { const cdir = resolve("./"); const cdirNoPrivate = cdir.replace(/^\/private/, ""); // for macos + // On Windows, normalize path to POSIX format to match args.path from Bun's resolver + const cdirPosix = cdir.replace(/\\/g, "/").replace(/^[a-zA-Z]:/, ""); const filterResolve = new RegExp( - `^(?!\\.\/main\\.ts)(?!${cdir}\/main\\.ts)(?!(?:/private)?${cdirNoPrivate}\/wrapper\\.mjs).*\\.ts$` + `^(?!\\.\/main\\.ts)(?!${cdir}\/main\\.ts)(?!${cdirPosix}\/main\\.ts)(?!(?:/private)?${cdirNoPrivate}\/wrapper\\.mjs).*\\.ts$` ); let cdirNodeModules = `${cdir}/node_modules/`; diff --git a/backend/windmill-worker/loader.py b/backend/windmill-worker/loader.py index 1361e06ead..c387e5e0fc 100644 --- a/backend/windmill-worker/loader.py +++ b/backend/windmill-worker/loader.py @@ -51,25 +51,32 @@ class WindmillFinder(MetaPathFinder): url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}.py{query_params}" req = urllib.request.Request(url, None, headers) - try: - req_start = time.time() - with urllib.request.urlopen(req) as response: - os.makedirs(folder, exist_ok=True) - r = response.read().decode("utf-8") - if r == "WINDMILL_IS_FOLDER": - return ModuleSpec(name, WindmillLoader(name)) - with open(fullpath, "w+") as f: - f.write(r) - return ModuleSpec(name, SourceFileLoader(name, fullpath)) - except urllib.error.HTTPError as e: - duration = time.time() - req_start - if e.code != 404: - print(f"Error fetching script {script_path}: HTTP {e.code} - {e.reason} - {duration}s") - return ModuleSpec(name, WindmillLoader(name)) - except Exception as e: - duration = time.time() - req_start - print(f"Error fetching script {script_path}: {e} - {duration}s") - return ModuleSpec(name, WindmillLoader(name)) + + for attempt in range(4): # 0, 1, 2, 3 = up to 3 retries + try: + req_start = time.time() + with urllib.request.urlopen(req) as response: + os.makedirs(folder, exist_ok=True) + r = response.read().decode("utf-8") + if r == "WINDMILL_IS_FOLDER": + return ModuleSpec(name, WindmillLoader(name)) + with open(fullpath, "w+") as f: + f.write(r) + return ModuleSpec(name, SourceFileLoader(name, fullpath)) + except urllib.error.HTTPError as e: + duration = time.time() - req_start + if e.code != 404: + print(f"Error fetching script {script_path}: HTTP {e.code} - {e.reason} - {duration}s") + return ModuleSpec(name, WindmillLoader(name)) + except Exception as e: + duration = time.time() - req_start + # Check if this is errno 104 (Connection reset by peer) and we have retries left + if (hasattr(e, 'errno') and e.errno == 104) and attempt < 3: + print(f"Connection reset (errno 104) fetching script {script_path}, retrying in 3s (attempt {attempt + 1}/3)") + time.sleep(3) + continue + print(f"Error fetching script {script_path}: {e} - {duration}s") + return ModuleSpec(name, WindmillLoader(name)) diff --git a/backend/windmill-worker/nsjail/run.csharp.config.proto b/backend/windmill-worker/nsjail/run.csharp.config.proto index d9f53e3b12..11e991940c 100644 --- a/backend/windmill-worker/nsjail/run.csharp.config.proto +++ b/backend/windmill-worker/nsjail/run.csharp.config.proto @@ -44,8 +44,8 @@ mount { } mount { - src: "/opt/dotnet-sdk/bin" - dst: "/opt/dotnet-sdk/bin" + src: "/usr/share/dotnet" + dst: "/usr/share/dotnet" is_bind: true } diff --git a/backend/windmill-worker/nsjail/run.powershell.config.proto b/backend/windmill-worker/nsjail/run.powershell.config.proto index f7580b7aa7..87b6abda21 100644 --- a/backend/windmill-worker/nsjail/run.powershell.config.proto +++ b/backend/windmill-worker/nsjail/run.powershell.config.proto @@ -78,8 +78,8 @@ mount { } mount { - src: "{JOB_DIR}/wrapper.sh" - dst: "/tmp/wrapper.sh" + src: "{JOB_DIR}/wrapper.ps1" + dst: "/tmp/wrapper.ps1" is_bind: true mandatory: false } diff --git a/backend/windmill-worker/src/ai/image_handler.rs b/backend/windmill-worker/src/ai/image_handler.rs new file mode 100644 index 0000000000..c668ded17f --- /dev/null +++ b/backend/windmill-worker/src/ai/image_handler.rs @@ -0,0 +1,68 @@ +use base64::Engine; +use futures; +use ulid; +use windmill_common::{client::AuthedClient, error::Error, s3_helpers::S3Object}; +use windmill_queue::MiniPulledJob; + +/// Upload image to S3 and return S3Object +pub async fn upload_image_to_s3( + base64_image: &str, + job: &MiniPulledJob, + client: &AuthedClient, +) -> Result { + let image_bytes = base64::engine::general_purpose::STANDARD + .decode(base64_image) + .map_err(|e| Error::internal_err(format!("Failed to decode base64 image: {}", e)))?; + + // Generate unique S3 key + let unique_id = ulid::Ulid::new().to_string(); + let s3_key = format!("ai_images/{}/{}.png", job.id, unique_id); + + // Create byte stream + let byte_stream = futures::stream::once(async move { + Ok::<_, std::convert::Infallible>(bytes::Bytes::from(image_bytes)) + }); + + // Upload to S3 + client + .upload_s3_file( + &job.workspace_id, + s3_key.clone(), + None, // storage - use default + byte_stream, + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to upload image to S3: {}", e)))?; + + Ok(S3Object { + s3: s3_key, + storage: None, + filename: Some("generated_image.png".to_string()), + presigned: None, + }) +} + +/// Download an S3 image and convert it to a base64 data URL +pub async fn download_and_encode_s3_image( + image: &S3Object, + client: &AuthedClient, + workspace_id: &str, +) -> Result<(String, String), Error> { + // Download the image from S3 + let image_bytes = client + .download_s3_file(workspace_id, &image.s3, image.storage.clone()) + .await + .map_err(|e| Error::internal_err(format!("Failed to download S3 image: {}", e)))?; + + // Encode as base64 data URL + let base64_data = base64::engine::general_purpose::STANDARD.encode(&image_bytes); + + // Determine MIME type using mime_guess from file extension, with PNG as fallback + let mime_type = mime_guess::from_path(&image.s3).first(); + let mime_type = mime_type + .as_ref() + .map(|mime| mime.essence_str()) + .unwrap_or("image/png"); + + Ok((mime_type.to_string(), base64_data)) +} diff --git a/backend/windmill-worker/src/ai/mod.rs b/backend/windmill-worker/src/ai/mod.rs new file mode 100644 index 0000000000..4ad67b9e6e --- /dev/null +++ b/backend/windmill-worker/src/ai/mod.rs @@ -0,0 +1,10 @@ +// AI executor module structure +// This module will contain all AI-related execution logic + +pub mod image_handler; +pub mod providers; +pub mod query_builder; +pub mod sse; +pub mod tools; +pub mod types; +pub mod utils; diff --git a/backend/windmill-worker/src/ai/providers/google_ai.rs b/backend/windmill-worker/src/ai/providers/google_ai.rs new file mode 100644 index 0000000000..b94d4f29e5 --- /dev/null +++ b/backend/windmill-worker/src/ai/providers/google_ai.rs @@ -0,0 +1,293 @@ +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json; +use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error}; + +use crate::ai::{ + image_handler::download_and_encode_s3_image, + query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, + types::*, +}; + +// Google AI/Gemini-specific types +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GeminiInlineData { + #[serde(rename = "mimeType")] + pub mime_type: String, + pub data: String, +} + +#[derive(Serialize, Deserialize, Clone)] +#[serde(untagged)] +pub enum GeminiPart { + Text { text: String }, + InlineData { inline_data: GeminiInlineData }, + FunctionCall { function_call: GeminiFunctionCall }, + FunctionResponse { function_response: GeminiFunctionResponse }, +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct GeminiFunctionCall { + pub name: String, + pub args: serde_json::Value, +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct GeminiFunctionResponse { + pub name: String, + pub response: serde_json::Value, +} + +#[derive(Serialize)] +pub struct GeminiContent { + pub parts: Vec, +} + +#[derive(Serialize)] +pub struct GeminiImageRequest { + #[serde(skip_serializing_if = "Option::is_none")] + pub contents: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub instances: Option>, +} + +#[derive(Serialize)] +pub struct GeminiPredictContent { + pub prompt: String, +} + +#[derive(Deserialize)] +pub struct GeminiImageResponse { + #[serde(skip_serializing_if = "Option::is_none")] + pub candidates: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub predictions: Option>, +} + +#[derive(Deserialize)] +pub struct GeminiCandidate { + pub content: GeminiResponseContent, +} + +#[derive(Deserialize)] +pub struct GeminiPredictCandidate { + #[serde(rename = "bytesBase64Encoded")] + pub bytes_base64_encoded: String, // base64 encoded image +} + +#[derive(Deserialize)] +pub struct GeminiResponseContent { + pub parts: Vec, +} + +#[derive(Deserialize)] +pub struct GeminiResponsePart { + #[serde(skip_serializing_if = "Option::is_none")] + #[allow(dead_code)] + pub text: Option, + #[serde(rename = "inlineData", skip_serializing_if = "Option::is_none")] + pub inline_data: Option, + #[serde(rename = "functionCall", skip_serializing_if = "Option::is_none")] + #[allow(dead_code)] + pub function_call: Option, +} + +pub struct GoogleAIQueryBuilder; + +impl GoogleAIQueryBuilder { + pub fn new() -> Self { + Self + } + + async fn build_image_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result { + let is_imagen = args.model.contains("imagen"); + + let request = if is_imagen { + // For Imagen models, use simple prompt format + GeminiImageRequest { + instances: Some(vec![GeminiPredictContent { + prompt: args.user_message.trim().to_string(), + }]), + contents: None, + } + } else { + // For Gemini models with image generation, build parts + let mut parts = vec![GeminiPart::Text { text: args.user_message.trim().to_string() }]; + + if let Some(system_prompt) = args.system_prompt { + parts.insert( + 0, + GeminiPart::Text { text: format!("SYSTEM PROMPT: {}", system_prompt.trim()) }, + ); + } + + // Add input images if provided + if let Some(images) = args.images { + for image in images.iter() { + if !image.s3.is_empty() { + let (mime_type, image_bytes) = + download_and_encode_s3_image(image, client, workspace_id).await?; + parts.push(GeminiPart::InlineData { + inline_data: GeminiInlineData { + mime_type: mime_type, + data: image_bytes, + }, + }); + } + } + } + + GeminiImageRequest { instances: None, contents: Some(vec![GeminiContent { parts }]) } + }; + + serde_json::to_string(&request) + .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) + } +} + +#[async_trait] +impl QueryBuilder for GoogleAIQueryBuilder { + fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool { + // Google AI supports tools only for text output + matches!(output_type, OutputType::Text) + } + + fn supports_streaming(&self) -> bool { + // Google AI supports streaming for text output + true + } + + async fn build_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + stream: bool, + ) -> Result { + match args.output_type { + OutputType::Text => { + // For text output, use OpenAI-compatible format + let openai_builder = super::openai::OpenAIQueryBuilder::new(AIProvider::GoogleAI); + openai_builder + .build_request(args, client, workspace_id, stream) + .await + } + OutputType::Image => self.build_image_request(args, client, workspace_id).await, + } + } + + async fn parse_response(&self, response: reqwest::Response) -> Result { + let url = response.url().path(); + + // For chat completions (text), use OpenAI parser + if url.contains("/chat/completions") { + let openai_builder = super::openai::OpenAIQueryBuilder::new(AIProvider::GoogleAI); + return openai_builder.parse_response(response).await; + } + + // Check if this is an image generation response + if url.contains(":predict") || url.contains(":generateContent") { + let response_text = response + .text() + .await + .map_err(|e| Error::internal_err(format!("Failed to read response text: {}", e)))?; + + let gemini_response: GeminiImageResponse = serde_json::from_str(&response_text) + .map_err(|e| { + Error::internal_err(format!( + "Failed to parse Gemini response: {}. Raw response: {}", + e, response_text + )) + })?; + + // Find image data in response + let image_data = gemini_response + .candidates + .as_ref() + .and_then(|candidates| { + candidates.iter().find_map(|candidate| { + candidate + .content + .parts + .iter() + .find_map(|part| part.inline_data.as_ref().map(|data| &data.data)) + }) + }) + .or_else(|| { + gemini_response + .predictions + .as_ref() + .and_then(|predictions| { + predictions + .iter() + .find_map(|prediction| Some(&prediction.bytes_base64_encoded)) + }) + }); + + if let Some(base64_image) = image_data { + Ok(ParsedResponse::Image { base64_data: base64_image.clone() }) + } else { + Err(Error::internal_err( + "No image data received from Gemini".to_string(), + )) + } + } else { + // This should not happen as we use OpenAI format for text + Err(Error::internal_err( + "Unexpected text response in Google AI parser".to_string(), + )) + } + } + + async fn parse_streaming_response( + &self, + response: reqwest::Response, + stream_event_processor: StreamEventProcessor, + ) -> Result { + let openai_builder = super::openai::OpenAIQueryBuilder::new(AIProvider::GoogleAI); + openai_builder + .parse_streaming_response(response, stream_event_processor) + .await + } + + fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String { + match output_type { + OutputType::Text => format!("{}/chat/completions", base_url), // Use OpenAI-compatible endpoint + OutputType::Image => { + // For image generation, build the full URL with model name + let url_suffix = if model.contains("imagen") { + "predict" + } else { + "generateContent" + }; + format!( + "https://generativelanguage.googleapis.com/v1beta/models/{}:{}", + model, url_suffix + ) + } + } + } + + fn get_auth_headers( + &self, + api_key: &str, + _base_url: &str, + output_type: &OutputType, + ) -> Vec<(&'static str, String)> { + match output_type { + OutputType::Text => { + // For text output, use Bearer token (OpenAI-compatible) + vec![("Authorization", format!("Bearer {}", api_key))] + } + OutputType::Image => { + // For image generation, use Google API key header + vec![("x-goog-api-key", api_key.to_string())] + } + } + } +} diff --git a/backend/windmill-worker/src/ai/providers/mod.rs b/backend/windmill-worker/src/ai/providers/mod.rs new file mode 100644 index 0000000000..13cf766e28 --- /dev/null +++ b/backend/windmill-worker/src/ai/providers/mod.rs @@ -0,0 +1,3 @@ +pub mod google_ai; +pub mod openai; +pub mod openrouter; diff --git a/backend/windmill-worker/src/ai/providers/openai.rs b/backend/windmill-worker/src/ai/providers/openai.rs new file mode 100644 index 0000000000..d764cc11ad --- /dev/null +++ b/backend/windmill-worker/src/ai/providers/openai.rs @@ -0,0 +1,433 @@ +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json; +use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error}; + +use crate::ai::{ + image_handler::download_and_encode_s3_image, + query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, + sse::{OpenAISSEParser, SSEParser}, + types::*, + utils::is_claude_model, +}; + +// OpenAI-specific types +#[derive(Deserialize, Serialize, Clone, Debug)] +pub struct OpenAIFunction { + pub name: String, + pub arguments: String, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +pub struct OpenAIToolCall { + pub id: String, + pub function: OpenAIFunction, + pub r#type: String, +} + +#[derive(Deserialize)] +pub struct OpenAIChoice { + pub message: OpenAIMessage, +} + +#[derive(Deserialize)] +pub struct OpenAIResponse { + pub choices: Vec, +} + +#[derive(Serialize)] +pub struct ImageGenerationTool { + pub r#type: String, + pub quality: Option, + pub background: Option, +} + +// Input content for image generation - supports both text and images +#[derive(Serialize, Clone, Debug)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ImageGenerationContent { + #[serde(rename = "input_text")] + InputText { text: String }, + #[serde(rename = "input_image")] + InputImage { image_url: String }, +} + +#[derive(Serialize)] +pub struct ImageGenerationMessage { + pub role: String, + pub content: Vec, +} + +#[derive(Serialize)] +pub struct ImageGenerationRequest<'a> { + pub model: &'a str, + pub input: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub instructions: Option<&'a str>, + pub tools: Vec, +} + +#[derive(Deserialize)] +pub struct OpenAIImageResponse { + pub output: Vec, +} + +#[derive(Deserialize)] +pub struct OpenAIImageOutput { + pub r#type: String, // Expected to be "image_generation_call" + pub status: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub result: Option, // Base64 encoded image, None if not completed +} + +#[derive(Serialize, Debug)] +#[serde(rename_all = "lowercase")] +pub enum ToolChoice { + #[allow(dead_code)] + Auto, + Required, +} + +#[derive(Serialize)] +pub struct OpenAIRequest<'a> { + pub model: &'a str, + pub messages: &'a [OpenAIMessage], + #[serde(skip_serializing_if = "Option::is_none")] + pub tools: Option<&'a [ToolDef]>, + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_completion_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub response_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_choice: Option, + pub stream: bool, +} + +pub struct OpenAIQueryBuilder { + provider_kind: AIProvider, +} + +impl OpenAIQueryBuilder { + pub fn new(provider_kind: AIProvider) -> Self { + Self { provider_kind } + } + + pub async fn prepare_messages_for_api( + &self, + messages: &[OpenAIMessage], + client: &AuthedClient, + workspace_id: &str, + ) -> Result, Error> { + let mut prepared_messages = Vec::new(); + + for message in messages { + let mut prepared_message = message.clone(); + + if let Some(content) = &message.content { + match content { + OpenAIContent::Text(text) => { + prepared_message.content = Some(OpenAIContent::Text(text.clone())); + } + OpenAIContent::Parts(parts) => { + let mut prepared_content = Vec::new(); + + for part in parts { + match part { + ContentPart::S3Object { s3_object } => { + // Convert S3Object to base64 image URL + let (mime_type, image_bytes) = download_and_encode_s3_image( + s3_object, + client, + workspace_id, + ) + .await?; + prepared_content.push(ContentPart::ImageUrl { + image_url: ImageUrlData { + url: format!( + "data:{};base64,{}", + mime_type, image_bytes + ), + }, + }); + } + other => { + // Keep Text and ImageUrl as-is + prepared_content.push(other.clone()); + } + } + } + + prepared_message.content = Some(OpenAIContent::Parts(prepared_content)); + } + } + } + + prepared_messages.push(prepared_message); + } + + Ok(prepared_messages) + } + + async fn build_text_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + stream: bool, + ) -> Result { + let prepared_messages = self + .prepare_messages_for_api(args.messages, client, workspace_id) + .await?; + + // Check if we need to add response_format for structured output + let has_output_properties = args + .output_schema + .and_then(|schema| schema.properties.as_ref()) + .map(|props| !props.is_empty()) + .unwrap_or(false); + + let response_format = if has_output_properties && args.output_schema.is_some() { + let schema = args.output_schema.unwrap(); + let strict_schema = schema.clone().make_strict(); + Some(ResponseFormat { + r#type: "json_schema".to_string(), + json_schema: JsonSchemaFormat { + name: "structured_output".to_string(), + schema: strict_schema, + strict: Some(true), + }, + }) + } else { + None + }; + + let is_claude_model = is_claude_model(&args.model); + // Force usage of structured output tool for Claude models when structured output provided + let tool_choice = if is_claude_model && response_format.is_some() { + Some(ToolChoice::Required) + } else { + None + }; + + let request = OpenAIRequest { + model: args.model, + messages: &prepared_messages, + tools: args.tools, + temperature: args.temperature, + max_completion_tokens: args.max_tokens, + response_format, + tool_choice, + stream, + }; + + serde_json::to_string(&request) + .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) + } + + async fn build_image_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result { + // Build content array with text and optional image + let mut content = + vec![ImageGenerationContent::InputText { text: args.user_message.to_string() }]; + + // Add images if provided + if let Some(images) = args.images { + for image in images.iter() { + if !image.s3.is_empty() { + let (mime_type, image_bytes) = + download_and_encode_s3_image(image, client, workspace_id).await?; + content.push(ImageGenerationContent::InputImage { + image_url: format!("data:{};base64,{}", mime_type, image_bytes), + }); + } + } + } + + // Build the request with tools if provided + let tools = vec![ImageGenerationTool { + r#type: "image_generation".to_string(), + quality: Some("low".to_string()), + background: None, + }]; + + // TODO: OpenAI's image generation API doesn't support custom tools in the same way as chat completions + // This would require a different approach, potentially using chat completions with image output + // For now, we'll use the standard image generation without custom tools + + let image_request = ImageGenerationRequest { + model: args.model, + input: vec![ImageGenerationMessage { role: "user".to_string(), content }], + instructions: args.system_prompt, + tools, + }; + + serde_json::to_string(&image_request) + .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) + } +} + +#[async_trait] +impl QueryBuilder for OpenAIQueryBuilder { + fn supports_tools_with_output_type(&self, _output_type: &OutputType) -> bool { + // OpenAI supports tools for both text and image output + true + } + + fn supports_streaming(&self) -> bool { + // OpenAI supports streaming for text output + true + } + + async fn build_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + stream: bool, + ) -> Result { + match args.output_type { + OutputType::Text => { + self.build_text_request(args, client, workspace_id, stream) + .await + } + OutputType::Image => self.build_image_request(args, client, workspace_id).await, + } + } + + async fn parse_response(&self, response: reqwest::Response) -> Result { + // Check if this is an image response + let url = response.url().path(); + if url.contains("/responses") { + // Parse image generation response + let response_text = response + .text() + .await + .map_err(|e| Error::internal_err(format!("Failed to read response text: {}", e)))?; + + let image_response: OpenAIImageResponse = serde_json::from_str(&response_text) + .map_err(|e| { + Error::internal_err(format!( + "Failed to parse OpenAI image response: {}. Raw response: {}", + e, response_text + )) + })?; + + // Find the first completed image generation output + let image_generation_call = image_response + .output + .iter() + .find(|output| { + output.r#type == "image_generation_call" && output.status == "completed" + }) + .and_then(|output| output.result.as_ref()); + + if let Some(base64_image) = image_generation_call { + Ok(ParsedResponse::Image { base64_data: base64_image.clone() }) + } else { + Err(Error::internal_err( + "No completed image output received from OpenAI".to_string(), + )) + } + } else { + // Parse text/chat completion response + let openai_response: OpenAIResponse = response + .json() + .await + .map_err(|e| Error::internal_err(format!("Failed to parse response: {}", e)))?; + + let first_choice = openai_response + .choices + .into_iter() + .next() + .ok_or_else(|| Error::internal_err("No response from API"))?; + + Ok(ParsedResponse::Text { + content: first_choice.message.content.map(|c| match c { + OpenAIContent::Text(text) => text, + OpenAIContent::Parts(parts) => { + // Extract text from parts + parts + .into_iter() + .filter_map(|part| match part { + ContentPart::Text { text } => Some(text), + _ => None, + }) + .collect::>() + .join(" ") + } + }), + tool_calls: first_choice.message.tool_calls.unwrap_or_default(), + events_str: None, + }) + } + } + + async fn parse_streaming_response( + &self, + response: reqwest::Response, + stream_event_processor: StreamEventProcessor, + ) -> Result { + let mut openai_sse_parser = OpenAISSEParser::new(stream_event_processor); + openai_sse_parser.parse_events(response).await?; + + let OpenAISSEParser { + accumulated_content, + accumulated_tool_calls, + mut events_str, + stream_event_processor, + } = openai_sse_parser; + + // Process streaming events with error handling + + for tool_call in accumulated_tool_calls.values() { + let event = StreamingEvent::ToolCallArguments { + call_id: tool_call.id.clone(), + function_name: tool_call.function.name.clone(), + arguments: tool_call.function.arguments.clone(), + }; + stream_event_processor.send(event, &mut events_str).await?; + } + + Ok(ParsedResponse::Text { + content: if accumulated_content.is_empty() { + None + } else { + Some(accumulated_content) + }, + tool_calls: accumulated_tool_calls.into_values().collect(), + events_str: Some(events_str), + }) + } + + fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String { + let path = match output_type { + OutputType::Text => "chat/completions", + OutputType::Image => "responses", + }; + + if self.provider_kind.is_azure_openai(base_url) { + AIProvider::build_azure_openai_url(base_url, model, path) + } else { + format!("{}/{}", base_url, path) + } + } + + fn get_auth_headers( + &self, + api_key: &str, + base_url: &str, + _output_type: &OutputType, + ) -> Vec<(&'static str, String)> { + if self.provider_kind.is_azure_openai(base_url) { + vec![("api-key", api_key.to_string())] + } else { + vec![("Authorization", format!("Bearer {}", api_key))] + } + } +} diff --git a/backend/windmill-worker/src/ai/providers/openrouter.rs b/backend/windmill-worker/src/ai/providers/openrouter.rs new file mode 100644 index 0000000000..9e22a63552 --- /dev/null +++ b/backend/windmill-worker/src/ai/providers/openrouter.rs @@ -0,0 +1,220 @@ +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json; +use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error}; + +use crate::ai::{ + providers::openai::{OpenAIQueryBuilder, OpenAIResponse}, + query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, + types::*, +}; + +// OpenRouter-specific types +#[derive(Serialize)] +pub struct OpenRouterChatRequest<'a> { + pub model: &'a str, + pub messages: &'a [OpenAIMessage], + #[serde(skip_serializing_if = "Option::is_none")] + pub tools: Option<&'a [ToolDef]>, + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_completion_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub response_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub modalities: Option>, +} + +#[derive(Deserialize)] +pub struct OpenRouterImageResponse { + pub choices: Vec, +} + +#[derive(Deserialize)] +pub struct OpenRouterImageChoice { + pub message: OpenRouterImageResponseMessage, +} + +#[derive(Deserialize)] +pub struct OpenRouterImageResponseMessage { + #[serde(skip_serializing_if = "Option::is_none")] + pub images: Option>, +} + +#[derive(Deserialize)] +pub struct OpenRouterImageData { + pub image_url: OpenRouterImageUrl, +} + +#[derive(Deserialize)] +pub struct OpenRouterImageUrl { + pub url: String, // data:image/png;base64,... format +} + +pub struct OpenRouterQueryBuilder { + // OpenRouter uses OpenAI-compatible API, so we delegate most work to OpenAI builder + openai_builder: OpenAIQueryBuilder, +} + +impl OpenRouterQueryBuilder { + pub fn new() -> Self { + Self { openai_builder: OpenAIQueryBuilder::new(AIProvider::OpenRouter) } + } +} + +#[async_trait] +impl QueryBuilder for OpenRouterQueryBuilder { + fn supports_tools_with_output_type(&self, _output_type: &OutputType) -> bool { + // OpenRouter supports tools for both text and image output (via OpenAI-compatible API) + true + } + + fn supports_streaming(&self) -> bool { + // OpenRouter supports streaming for text output + true + } + + async fn build_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + stream: bool, + ) -> Result { + match args.output_type { + OutputType::Text => { + // For text, use standard OpenAI format without modalities + self.openai_builder + .build_request(args, client, workspace_id, stream) + .await + } + OutputType::Image => { + // For image generation, we need to add modalities field + // First, prepare the messages using the OpenAI builder's logic + let openai_builder = &self.openai_builder; + let prepared_messages = openai_builder + .prepare_messages_for_api(args.messages, client, workspace_id) + .await?; + + // Check if we need to add response_format for structured output + let has_output_properties = args + .output_schema + .and_then(|schema| schema.properties.as_ref()) + .map(|props| !props.is_empty()) + .unwrap_or(false); + + let response_format = if has_output_properties && args.output_schema.is_some() { + let schema = args.output_schema.unwrap(); + let strict_schema = schema.clone().make_strict(); + Some(ResponseFormat { + r#type: "json_schema".to_string(), + json_schema: JsonSchemaFormat { + name: "structured_output".to_string(), + schema: strict_schema, + strict: Some(true), + }, + }) + } else { + None + }; + + // Build OpenRouter-specific request with modalities + let request = OpenRouterChatRequest { + model: args.model, + messages: &prepared_messages, + tools: args.tools, + temperature: args.temperature, + max_completion_tokens: args.max_tokens, + response_format, + modalities: Some(vec!["image", "text"]), + }; + + serde_json::to_string(&request) + .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) + } + } + } + + async fn parse_response(&self, response: reqwest::Response) -> Result { + let response_text = response + .text() + .await + .map_err(|e| Error::internal_err(format!("Failed to read response text: {}", e)))?; + + // First try to parse as OpenRouter image response + if let Ok(image_response) = serde_json::from_str::(&response_text) + { + // Extract base64 image from the first choice + let image_url = image_response + .choices + .get(0) + .and_then(|choice| choice.message.images.as_ref()) + .and_then(|images| images.get(0)) + .map(|image| &image.image_url.url); + + if let Some(data_url) = image_url { + // Extract base64 data from data URL format: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... + if let Some(base64_start) = data_url.find("base64,") { + let base64_data = &data_url[base64_start + 7..]; // Skip "base64," prefix + return Ok(ParsedResponse::Image { base64_data: base64_data.to_string() }); + } + } + } + + // If not an image response or parsing failed, try as regular OpenAI response + let openai_response: OpenAIResponse = + serde_json::from_str(&response_text).map_err(|e| { + Error::internal_err(format!( + "Failed to parse response: {}. Raw response: {}", + e, response_text + )) + })?; + + let first_choice = openai_response + .choices + .into_iter() + .next() + .ok_or_else(|| Error::internal_err("No response from API"))?; + + Ok(ParsedResponse::Text { + content: first_choice.message.content.map(|c| match c { + OpenAIContent::Text(text) => text, + OpenAIContent::Parts(parts) => parts + .into_iter() + .filter_map(|part| match part { + ContentPart::Text { text } => Some(text), + _ => None, + }) + .collect::>() + .join(" "), + }), + tool_calls: first_choice.message.tool_calls.unwrap_or_default(), + events_str: None, + }) + } + + async fn parse_streaming_response( + &self, + response: reqwest::Response, + stream_event_processor: StreamEventProcessor, + ) -> Result { + self.openai_builder + .parse_streaming_response(response, stream_event_processor) + .await + } + + fn get_endpoint(&self, base_url: &str, _model: &str, _output_type: &OutputType) -> String { + // OpenRouter uses the same endpoint for both text and image generation + format!("{}/chat/completions", base_url) + } + + fn get_auth_headers( + &self, + api_key: &str, + _base_url: &str, + _output_type: &OutputType, + ) -> Vec<(&'static str, String)> { + vec![("Authorization", format!("Bearer {}", api_key))] + } +} diff --git a/backend/windmill-worker/src/ai/query_builder.rs b/backend/windmill-worker/src/ai/query_builder.rs new file mode 100644 index 0000000000..e332c21f73 --- /dev/null +++ b/backend/windmill-worker/src/ai/query_builder.rs @@ -0,0 +1,166 @@ +use async_trait::async_trait; +use windmill_common::{ + client::AuthedClient, error::Error, s3_helpers::S3Object, worker::Connection, +}; +use windmill_queue::MiniPulledJob; + +use crate::{ + ai::{ + providers::{ + google_ai::GoogleAIQueryBuilder, + openai::{OpenAIQueryBuilder, OpenAIToolCall}, + openrouter::OpenRouterQueryBuilder, + }, + types::*, + }, + job_logger::append_result_stream, +}; + +/// Arguments for building an AI request +pub struct BuildRequestArgs<'a> { + pub messages: &'a [OpenAIMessage], + pub tools: Option<&'a [ToolDef]>, + pub model: &'a str, + pub temperature: Option, + pub max_tokens: Option, + pub output_schema: Option<&'a OpenAPISchema>, + pub output_type: &'a OutputType, + pub system_prompt: Option<&'a str>, + pub user_message: &'a str, + pub images: Option<&'a [S3Object]>, +} + +/// Response from AI provider +pub enum ParsedResponse { + Text { content: Option, tool_calls: Vec, events_str: Option }, + Image { base64_data: String }, +} + +/// Trait for building provider-specific AI requests +#[async_trait] +pub trait QueryBuilder: Send + Sync { + /// Check if this provider supports tools with the given output type + fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool; + + /// Check if this provider supports streaming + fn supports_streaming(&self) -> bool; + + /// Build the request body for the provider + async fn build_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + stream: bool, + ) -> Result; + + /// Parse the response from the provider + async fn parse_response(&self, response: reqwest::Response) -> Result; + + /// Parse streaming response from the provider + async fn parse_streaming_response( + &self, + _response: reqwest::Response, + _stream_event_processor: StreamEventProcessor, + ) -> Result { + return Err(Error::internal_err( + "Missing implementation for parse_streaming_response for this provider".to_string(), + )); + } + + /// Get the API endpoint for this provider + fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String; + + /// Get the authentication headers for this provider + fn get_auth_headers( + &self, + api_key: &str, + base_url: &str, + output_type: &OutputType, + ) -> Vec<(&'static str, String)>; +} + +/// Factory function to create the appropriate query builder for a provider +pub fn create_query_builder(provider: &ProviderWithResource) -> Box { + use windmill_common::ai_providers::AIProvider; + + match provider.kind { + AIProvider::GoogleAI => Box::new(GoogleAIQueryBuilder::new()), + AIProvider::OpenRouter => Box::new(OpenRouterQueryBuilder::new()), + _ => Box::new(OpenAIQueryBuilder::new(provider.kind.clone())), // Pass provider kind for Azure handling + } +} + +pub struct StreamEventProcessor { + tx: tokio::sync::mpsc::Sender, + pub handle: Option>, +} + +impl Clone for StreamEventProcessor { + fn clone(&self) -> Self { + Self { tx: self.tx.clone(), handle: None } + } +} + +impl StreamEventProcessor { + pub fn new(conn: &Connection, job: &MiniPulledJob) -> Self { + let (tx, mut rx) = tokio::sync::mpsc::channel::(100); + let conn = conn.clone(); + let job_id = job.id.clone(); + let workspace_id = job.workspace_id.clone(); + let handle = tokio::spawn(async move { + let mut offset = -1; + while let Some(event) = rx.recv().await { + offset += 1; + match tokio::time::timeout( + std::time::Duration::from_secs(20), + append_result_stream(&conn, &workspace_id, &job_id, &event, offset), + ) + .await + { + Ok(res) => { + if let Err(err) = res { + tracing::error!("Failed to save stream event: {}", err); + } + } + Err(err) => { + tracing::error!("Did not manage to save stream event after 20 seconds, stopping stream event processor: {}", err); + break; + } + } + } + }); + + Self { tx, handle: Some(handle) } + } + + pub async fn send(&self, event: StreamingEvent, events_str: &mut String) -> Result<(), Error> { + match serde_json::to_string(&event) { + Ok(event_json) => { + let event_json = format!("{}\n", event_json); + events_str.push_str(&event_json); + if let Err(err) = self + .tx + .send(event_json.clone()) + .await + .map_err(|e| Error::internal_err(format!("Failed to send event: {}", e))) + { + tracing::error!( + "Failed to send event to stream event processor, skiping event: {}", + err + ); + } + + Ok(()) + } + Err(e) => Err(Error::internal_err(format!( + "Failed to serialize streaming event {:#?}, error is: {}", + event, e + ))), + } + } + + pub fn to_handle(self) -> Option> { + self.handle + } +} diff --git a/backend/windmill-worker/src/ai/sse.rs b/backend/windmill-worker/src/ai/sse.rs new file mode 100644 index 0000000000..fcafc9cd7e --- /dev/null +++ b/backend/windmill-worker/src/ai/sse.rs @@ -0,0 +1,158 @@ +use std::collections::HashMap; + +use reqwest::Response; +use serde::Deserialize; +use serde_json; +use tokio_stream::StreamExt; +use windmill_common::{error::Error, utils::rd_string}; + +use crate::ai::{ + providers::openai::{OpenAIFunction, OpenAIToolCall}, + query_builder::StreamEventProcessor, + types::StreamingEvent, +}; + +#[derive(Deserialize)] +pub struct OpenAIChoiceDeltaToolCallFunction { + pub name: Option, + pub arguments: Option, +} + +#[derive(Deserialize)] +pub struct OpenAIChoiceDeltaToolCall { + pub index: Option, + pub id: Option, + pub function: Option, +} + +#[derive(Deserialize)] +pub struct OpenAIChoiceDelta { + pub content: Option, + pub tool_calls: Option>, +} + +#[derive(Deserialize)] +pub struct OpenAIChoice { + pub delta: Option, +} + +#[derive(Deserialize)] +pub struct OpenAISSEEvent { + pub choices: Option>, +} + +pub trait SSEParser { + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error>; + + async fn parse_events(&mut self, response: Response) -> Result<(), Error> { + let mut stream = response.bytes_stream(); + let mut buffer = String::new(); + + while let Some(chunk_result) = stream.next().await { + let chunk = chunk_result + .map_err(|e| Error::internal_err(format!("Failed to read chunk: {}", e)))?; + + // Convert chunk to string and add to buffer + let chunk_str = String::from_utf8_lossy(&chunk); + buffer.push_str(&chunk_str); + + // Process complete lines from buffer + while let Some(newline_pos) = buffer.find("\n\n") { + let line = buffer.drain(..newline_pos + 2).collect::(); + let line = line.trim_end_matches('\n'); + + // Skip empty lines and comments + if line.is_empty() || line.starts_with(':') { + continue; + } + + // Parse SSE data field + if let Some(data) = line.strip_prefix("data: ") { + if data == "[DONE]" { + // OpenAI sends [DONE] to indicate end of stream + return Ok(()); + } + + self.parse_event_data(data).await?; + } + } + } + + Ok(()) + } +} + +pub struct OpenAISSEParser { + pub accumulated_content: String, + pub accumulated_tool_calls: HashMap, + pub events_str: String, + pub stream_event_processor: StreamEventProcessor, +} + +impl OpenAISSEParser { + pub fn new(stream_event_processor: StreamEventProcessor) -> Self { + Self { + accumulated_content: String::new(), + accumulated_tool_calls: HashMap::new(), + events_str: String::new(), + stream_event_processor, + } + } +} + +impl SSEParser for OpenAISSEParser { + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { + let event: OpenAISSEEvent = serde_json::from_str(data).map_err(|e| { + Error::internal_err(format!("Failed to parse SSE chunk {}: {}", data, e)) + })?; + + if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) { + if let Some(delta) = choices.remove(0).delta { + if let Some(content) = delta.content.filter(|s| !s.is_empty()) { + self.accumulated_content.push_str(&content); + let event = StreamingEvent::TokenDelta { content }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + } + + if let Some(tool_calls) = delta.tool_calls { + for (idx, tool_call) in tool_calls.into_iter().enumerate() { + let idx = tool_call.index.unwrap_or_else(|| idx as i64); + + if let Some(function) = tool_call.function { + if let Some(tool_call) = self.accumulated_tool_calls.get_mut(&idx) { + if let Some(arguments) = function.arguments { + tool_call.function.arguments += &arguments; + } + } else { + let fun_name = function.name.unwrap_or_default(); + let call_id = tool_call.id.unwrap_or_else(|| rd_string(24)); + let event = StreamingEvent::ToolCall { + call_id: call_id.clone(), + function_name: fun_name.clone(), + }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + self.accumulated_tool_calls.insert( + idx, + OpenAIToolCall { + id: call_id, + function: OpenAIFunction { + name: fun_name, + arguments: function.arguments.unwrap_or_default(), + }, + r#type: "function".to_string(), + }, + ); + } + } + } + } + } + } + + Ok(()) + } +} diff --git a/backend/windmill-worker/src/ai/tools.rs b/backend/windmill-worker/src/ai/tools.rs new file mode 100644 index 0000000000..d1280f0053 --- /dev/null +++ b/backend/windmill-worker/src/ai/tools.rs @@ -0,0 +1,754 @@ +use crate::ai::providers::openai::OpenAIToolCall; +use crate::ai::query_builder::StreamEventProcessor; +use crate::ai::types::*; +use crate::ai::utils::{ + add_message_to_conversation, execute_mcp_tool, get_step_name_from_flow, + update_flow_status_module_with_actions, update_flow_status_module_with_actions_success, + FlowContext, +}; +use crate::common::{error_to_value, OccupancyMetrics}; +use crate::result_processor::handle_non_flow_job_error; +use crate::worker_flow::{ + evaluate_input_transform, raw_script_to_payload, script_to_payload, JobPayloadWithTag, +}; +use crate::{ + create_job_dir, handle_queued_job, JobCompletedReceiver, JobCompletedSender, SendResult, + SendResultPayload, +}; +use anyhow::Context; +use mappable_rc::Marc; +use serde_json::value::RawValue; +use std::{collections::HashMap, sync::Arc}; +use uuid::Uuid; +use windmill_common::flows::InputTransform; +use windmill_common::jobs::JobPayload; +use windmill_common::mcp_client::{McpClient, McpToolSource}; +use windmill_common::{ + client::AuthedClient, + db::DB, + error::{to_anyhow, Error}, + flow_conversations::MessageType, + flow_status::AgentAction, + flows::FlowModuleValue, + worker::{to_raw_value, Connection}, +}; +use windmill_queue::{ + get_mini_pulled_job, push, JobCompleted, MiniCompletedJob, MiniPulledJob, PushArgs, + PushIsolationLevel, +}; + +/// Context for tool execution containing all required references and state +pub struct ToolExecutionContext<'a> { + // Database & connections + pub db: &'a DB, + pub conn: &'a Connection, + + // Job context + pub job: &'a MiniPulledJob, + pub parent_job: &'a Uuid, + pub summary: &'a Option<&'a str>, + + // Execution parameters + pub client: &'a AuthedClient, + pub worker_dir: &'a str, + pub base_internal_url: &'a str, + pub worker_name: &'a str, + pub hostname: &'a str, + + // Runtime state + pub occupancy_metrics: &'a mut OccupancyMetrics, + pub job_completed_tx: &'a JobCompletedSender, + pub killpill_rx: &'a mut tokio::sync::broadcast::Receiver<()>, + + // Optional streaming & chat + pub stream_event_processor: Option<&'a StreamEventProcessor>, + pub flow_context: &'a mut FlowContext, + pub previous_result: &'a Option>, + pub id_context: &'a Option, +} + +/// Execute all tool calls from an AI response +pub async fn execute_tool_calls( + mut ctx: ToolExecutionContext<'_>, + tool_calls: &[OpenAIToolCall], + tools: &[Tool], + mcp_clients: &HashMap>, + actions: &mut Vec, + final_events_str: &mut String, + structured_output_tool_name: &Option, +) -> Result<(Vec, Option, bool), Error> { + let mut messages = Vec::new(); + let mut used_structured_output_tool = false; + let mut final_content = None; + + for tool_call in tool_calls.iter() { + // Stream tool call progress + if let Some(stream_event_processor) = ctx.stream_event_processor { + let event = StreamingEvent::ToolExecution { + call_id: tool_call.id.clone(), + function_name: tool_call.function.name.clone(), + }; + stream_event_processor.send(event, final_events_str).await?; + } + + // Check if this is the structured output tool + if structured_output_tool_name + .as_ref() + .map_or(false, |name| tool_call.function.name == *name) + { + used_structured_output_tool = true; + messages.push(OpenAIMessage { + role: "tool".to_string(), + content: Some(OpenAIContent::Text( + "Successfully ran structured_output tool".to_string(), + )), + tool_call_id: Some(tool_call.id.clone()), + ..Default::default() + }); + messages.push(OpenAIMessage { + role: "assistant".to_string(), + content: Some(OpenAIContent::Text(tool_call.function.arguments.clone())), + agent_action: Some(AgentAction::Message {}), + ..Default::default() + }); + final_content = Some(OpenAIContent::Text(tool_call.function.arguments.clone())); + break; + } + + let tool = tools + .iter() + .find(|t| t.def.function.name == tool_call.function.name); + + if let Some(tool) = tool { + // Check if this is an MCP tool + if let Some(mcp_source) = &tool.mcp_source { + execute_mcp_tool_call( + &mut ctx, + tool_call, + mcp_clients, + mcp_source, + actions, + &mut messages, + final_events_str, + ) + .await?; + } else if tool.module.is_some() { + execute_windmill_tool( + &mut ctx, + tool_call, + tool, + actions, + &mut messages, + final_events_str, + ) + .await?; + } else { + return Err(Error::internal_err(format!( + "Tool type not supported: {}", + tool_call.function.name + ))); + } + } else { + return Err(Error::internal_err(format!( + "Tool not found: {}", + tool_call.function.name + ))); + } + } + + Ok((messages, final_content, used_structured_output_tool)) +} + +/// Execute an MCP tool call +async fn execute_mcp_tool_call( + ctx: &mut ToolExecutionContext<'_>, + tool_call: &OpenAIToolCall, + mcp_clients: &HashMap>, + mcp_source: &McpToolSource, + actions: &mut Vec, + messages: &mut Vec, + final_events_str: &mut String, +) -> Result<(), Error> { + let tool_result = + execute_mcp_tool(mcp_clients, mcp_source, &tool_call.function.arguments).await; + + let call_id = ulid::Ulid::new().into(); + let resource_path = &mcp_source.resource_path; + let tool_name = &tool_call.function.name; + let arguments = serde_json::from_str(&tool_call.function.arguments).ok(); + + actions.push(AgentAction::McpToolCall { + call_id, + function_name: tool_name.clone(), + resource_path: resource_path.clone(), + arguments: arguments.clone(), + }); + + match tool_result { + Ok(result) => { + let result_str = + serde_json::to_string_pretty(&result).unwrap_or_else(|_| result.to_string()); + + messages.push(OpenAIMessage { + role: "tool".to_string(), + content: Some(OpenAIContent::Text(result_str.clone())), + tool_call_id: Some(tool_call.id.clone()), + agent_action: Some(AgentAction::McpToolCall { + call_id, + function_name: tool_name.clone(), + resource_path: resource_path.clone(), + arguments: arguments.clone(), + }), + ..Default::default() + }); + + // Stream tool result + if let Some(stream_event_processor) = ctx.stream_event_processor { + let event = StreamingEvent::ToolResult { + call_id: tool_call.id.clone(), + function_name: tool_call.function.name.clone(), + result: result.to_string(), + success: true, + }; + stream_event_processor.send(event, final_events_str).await?; + } + + // Add tool message to conversation if chat_input_enabled + let content = format!("Used {} tool", tool_call.function.name); + add_tool_message_to_chat(ctx, None, &content, true).await; + } + Err(e) => { + let error_msg = format!("MCP tool error: {}", e); + tracing::error!("{}", error_msg); + + messages.push(OpenAIMessage { + role: "tool".to_string(), + content: Some(OpenAIContent::Text(error_msg.clone())), + tool_call_id: Some(tool_call.id.clone()), + agent_action: Some(AgentAction::McpToolCall { + call_id, + function_name: tool_name.clone(), + resource_path: resource_path.clone(), + arguments: arguments.clone(), + }), + ..Default::default() + }); + + // Stream tool error + if let Some(stream_event_processor) = ctx.stream_event_processor { + let event = StreamingEvent::ToolResult { + call_id: tool_call.id.clone(), + function_name: tool_name.clone(), + result: error_msg.clone(), + success: false, + }; + stream_event_processor.send(event, final_events_str).await?; + } + + // Add tool message to conversation if chat_input_enabled + add_tool_message_to_chat(ctx, None, &error_msg, false).await; + } + } + + Ok(()) +} + +/// Execute a Windmill tool (script or flow) +async fn execute_windmill_tool( + ctx: &mut ToolExecutionContext<'_>, + tool_call: &OpenAIToolCall, + tool: &Tool, + actions: &mut Vec, + messages: &mut Vec, + final_events_str: &mut String, +) -> Result<(), Error> { + // Regular Windmill tools must have a module + let tool_module = tool.module.as_ref().ok_or_else(|| { + Error::internal_err(format!("Tool {} has no module", tool_call.function.name)) + })?; + + let job_id = ulid::Ulid::new().into(); + actions.push(AgentAction::ToolCall { + job_id, + function_name: tool_call.function.name.clone(), + module_id: tool_module.id.clone(), + }); + + update_flow_status_module_with_actions(ctx.db, ctx.parent_job, actions).await?; + + let raw_tool_call_args = if tool_call.function.arguments.is_empty() { + "{}".to_string() + } else { + tool_call.function.arguments.clone() + }; + + let mut tool_call_args = serde_json::from_str::>>( + &raw_tool_call_args, + ) + .with_context(|| { + format!( + "Failed to parse tool call arguments for tool call {}: {}", + tool_call.function.name, tool_call.function.arguments + ) + })?; + + // Get input transforms given by the user and merge them with AI given args + let input_transforms = match tool_module.get_value()? { + FlowModuleValue::Script { input_transforms, .. } => input_transforms, + FlowModuleValue::RawScript { input_transforms, .. } => input_transforms, + FlowModuleValue::FlowScript { input_transforms, .. } => input_transforms, + _ => { + return Err(Error::internal_err(format!( + "Unsupported tool: {}", + tool_call.function.name + ))); + } + }; + + // Prepare context for transform evaluation + let last_result = Arc::new( + ctx.previous_result + .as_ref() + .cloned() + .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)), + ); + + let flow_inputs = ctx + .flow_context + .flow_inputs + .as_ref() + .map(|args| Marc::new(args.clone())); + + // Evaluate each input transform and merge with AI-provided args + for (key, transform) in input_transforms.iter() { + // We skip static empty / null values, those are the one the AI will fill in + if let InputTransform::Static { value } = transform { + let val = value.get().trim(); + if val.is_empty() || val == "null" { + continue; + } + } + let result = evaluate_input_transform::>( + transform, + last_result.clone(), + flow_inputs.clone(), + Some(ctx.client), + ctx.id_context.as_ref(), + ) + .await?; + + tool_call_args.insert(key.clone(), result); + } + + let job_payload = match tool_module.get_value()? { + FlowModuleValue::Script { path: script_path, hash: script_hash, tag_override, .. } => { + script_to_payload( + script_hash, + script_path, + ctx.db, + ctx.job, + tool_module, + tag_override, + tool_module.apply_preprocessor, + ) + .await? + } + FlowModuleValue::RawScript { + path, + content, + language, + lock, + tag, + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + .. + } => { + let path = path + .unwrap_or_else(|| format!("{}/tools/{}", ctx.job.runnable_path(), tool_module.id)); + + raw_script_to_payload( + path, + content, + language, + lock, + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + tool_module, + tag, + tool_module.delete_after_use.unwrap_or(false), + ) + } + FlowModuleValue::FlowScript { + id, + language, + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + tag, + .. + } => { + let path = format!("{}/tools/{}", ctx.job.runnable_path(), tool_module.id); + + let payload = JobPayloadWithTag { + payload: JobPayload::FlowScript { + id, + language, + custom_concurrency_key: custom_concurrency_key.clone(), + concurrent_limit, + concurrency_time_window_s, + cache_ttl: tool_module.cache_ttl.map(|x| x as i32), + dedicated_worker: None, + path, + }, + tag: tag.clone(), + delete_after_use: tool_module.delete_after_use.unwrap_or(false), + timeout: None, + on_behalf_of: None, + }; + payload + } + _ => { + return Err(Error::internal_err(format!( + "Unsupported tool: {}", + tool_call.function.name + ))); + } + }; + + let mut tx = ctx.db.begin().await?; + + let job_perms = + windmill_common::auth::get_job_perms(&mut *tx, &ctx.job.id, &ctx.job.workspace_id) + .await? + .map(|x| x.into()); + + let (email, permissioned_as) = if let Some(on_behalf_of) = job_payload.on_behalf_of.as_ref() { + (&on_behalf_of.email, on_behalf_of.permissioned_as.clone()) + } else { + ( + &ctx.job.permissioned_as_email, + ctx.job.permissioned_as.to_owned(), + ) + }; + + let job_priority = tool_module.priority.or(ctx.job.priority); + + let tx = PushIsolationLevel::Transaction(tx); + let (uuid, tx) = push( + ctx.db, + tx, + &ctx.job.workspace_id, + job_payload.payload, + PushArgs { args: &tool_call_args, extra: None }, + &ctx.job.created_by, + email, + permissioned_as, + Some(&format!("job-span-{}", ctx.job.id)), + None, + ctx.job.schedule_path(), + Some(ctx.job.id), + None, + None, + Some(job_id), + false, + false, + None, + ctx.job.visible_to_owner, + Some(ctx.job.tag.clone()), + job_payload.timeout, + None, + job_priority, + job_perms.as_ref(), + true, + None, + None, + ) + .await?; + + tx.commit().await?; + + let tool_job = get_mini_pulled_job(ctx.db, &uuid).await?; + + let Some(tool_job) = tool_job else { + return Err(Error::internal_err("Tool job not found".to_string())); + }; + + let tool_job = Arc::new(tool_job); + + let (inner_job_completed_tx, inner_job_completed_rx) = JobCompletedSender::new(ctx.conn, 1); + + let inner_job_completed_rx = inner_job_completed_rx.expect( + "inner_job_completed_tx should be set as agent jobs are not supported on agent workers", + ); + + // Spawn handle_queued_job on separate task to prevent tokio stack overflow + // Clone everything needed for the spawned task + let tool_job_spawn = tool_job.clone(); + let conn_spawn = ctx.conn.clone(); + let client_spawn = ctx.client.clone(); + let hostname_spawn = ctx.hostname.to_string(); + let worker_name_spawn = ctx.worker_name.to_string(); + let worker_dir_spawn = ctx.worker_dir.to_string(); + let base_internal_url_spawn = ctx.base_internal_url.to_string(); + let inner_job_completed_tx_spawn = inner_job_completed_tx.clone(); + let mut occupancy_metrics_spawn = ctx.occupancy_metrics.clone(); + let mut killpill_rx_spawn = ctx.killpill_rx.resubscribe(); + + // Spawn on separate tokio task with fresh stack + let join_handle = tokio::task::spawn(async move { + #[cfg(feature = "benchmark")] + let mut bench_spawn = windmill_common::bench::BenchmarkIter::new(); + + let job_dir = create_job_dir(&worker_dir_spawn, tool_job_spawn.id).await; + + let result = handle_queued_job( + tool_job_spawn, + None, + None, + None, + None, + &conn_spawn, + &client_spawn, + &hostname_spawn, + &worker_name_spawn, + &worker_dir_spawn, + &job_dir, + None, + &base_internal_url_spawn, + inner_job_completed_tx_spawn, + &mut occupancy_metrics_spawn, + &mut killpill_rx_spawn, + None, + #[cfg(feature = "benchmark")] + &mut bench_spawn, + ) + .await; + + // Return both result and updated metrics + (result, occupancy_metrics_spawn) + }); + + // Await the spawned task + let (handle_result, updated_occupancy) = join_handle + .await + .map_err(|e| Error::internal_err(format!("Tool execution task failed: {}", e)))?; + + // Merge occupancy metrics back + ctx.occupancy_metrics.total_duration_of_running_jobs = + updated_occupancy.total_duration_of_running_jobs; + + // Continue with match on handle_result + match handle_result { + Err(err) => { + handle_tool_execution_error( + ctx, + tool_call, + tool_module, + &MiniCompletedJob::from(tool_job), + job_id, + err, + messages, + final_events_str, + ) + .await?; + } + Ok(success) => { + handle_tool_execution_success( + ctx, + tool_call, + tool_module, + job_id, + success, + inner_job_completed_rx, + messages, + final_events_str, + ) + .await?; + } + } + + Ok(()) +} + +/// Handle tool execution error +async fn handle_tool_execution_error( + ctx: &mut ToolExecutionContext<'_>, + tool_call: &OpenAIToolCall, + tool_module: &windmill_common::flows::FlowModule, + tool_job: &MiniCompletedJob, + job_id: Uuid, + err: Error, + messages: &mut Vec, + final_events_str: &mut String, +) -> Result<(), Error> { + let err_string = format!("{}: {}", err.name(), err.to_string()); + let err_json = error_to_value(&err); + let _ = handle_non_flow_job_error( + ctx.db, + tool_job, + 0, + None, + err_string.clone(), + err_json, + ctx.worker_name, + ) + .await; + + let error_message = format!("Error running tool: {}", err_string); + messages.push(OpenAIMessage { + role: "tool".to_string(), + content: Some(OpenAIContent::Text(error_message.clone())), + tool_call_id: Some(tool_call.id.clone()), + agent_action: Some(AgentAction::ToolCall { + job_id, + function_name: tool_call.function.name.clone(), + module_id: tool_module.id.clone(), + }), + ..Default::default() + }); + + // Stream tool result (error case) + if let Some(stream_event_processor) = ctx.stream_event_processor { + let tool_result_event = StreamingEvent::ToolResult { + call_id: tool_call.id.clone(), + function_name: tool_call.function.name.clone(), + result: error_message.clone(), + success: false, + }; + stream_event_processor + .send(tool_result_event, final_events_str) + .await?; + } + + update_flow_status_module_with_actions_success(ctx.db, ctx.parent_job, false).await?; + + // Add tool message to conversation if chat_input_enabled (error case) + add_tool_message_to_chat(ctx, Some(job_id), &error_message, false).await; + + Ok(()) +} + +/// Handle tool execution success +async fn handle_tool_execution_success( + ctx: &mut ToolExecutionContext<'_>, + tool_call: &OpenAIToolCall, + tool_module: &windmill_common::flows::FlowModule, + job_id: Uuid, + success: bool, + inner_job_completed_rx: JobCompletedReceiver, + messages: &mut Vec, + final_events_str: &mut String, +) -> Result<(), Error> { + let send_result = inner_job_completed_rx.bounded_rx.try_recv().ok(); + + let result = if let Some(SendResult { + result: SendResultPayload::JobCompleted(JobCompleted { result, .. }), + .. + }) = send_result.as_ref() + { + ctx.job_completed_tx + .send(send_result.as_ref().unwrap().result.clone(), true) + .await + .map_err(to_anyhow)?; + result + } else { + if let Some(send_result) = send_result { + ctx.job_completed_tx + .send(send_result.result, true) + .await + .map_err(to_anyhow)?; + } + return Err(Error::internal_err( + "Tool job completed but no result".to_string(), + )); + }; + + messages.push(OpenAIMessage { + role: "tool".to_string(), + content: Some(OpenAIContent::Text(result.get().to_string())), + tool_call_id: Some(tool_call.id.clone()), + agent_action: Some(AgentAction::ToolCall { + job_id, + function_name: tool_call.function.name.clone(), + module_id: tool_module.id.clone(), + }), + ..Default::default() + }); + + // Stream tool result (success case) + if let Some(stream_event_processor) = ctx.stream_event_processor { + let tool_result_event = StreamingEvent::ToolResult { + call_id: tool_call.id.clone(), + function_name: tool_call.function.name.clone(), + result: result.get().to_string(), + success: true, + }; + stream_event_processor + .send(tool_result_event, final_events_str) + .await?; + } + + update_flow_status_module_with_actions_success(ctx.db, ctx.parent_job, success).await?; + + // Add tool message to conversation if chat_input_enabled + let content = if success { + format!("Used {} tool", tool_call.function.name) + } else { + format!("Error executing {}", tool_call.function.name) + }; + + add_tool_message_to_chat(ctx, Some(job_id), &content, success).await; + + Ok(()) +} + +/// Add tool message to conversation if chat is enabled +async fn add_tool_message_to_chat( + ctx: &mut ToolExecutionContext<'_>, + tool_job_id: Option, + content: &str, + success: bool, +) { + let chat_enabled = ctx + .flow_context + .flow_status + .as_ref() + .and_then(|fs| fs.chat_input_enabled) + .unwrap_or(false); + if chat_enabled { + if let Some(memory_id) = ctx + .flow_context + .flow_status + .as_ref() + .and_then(|fs| fs.memory_id) + { + let db_clone = ctx.db.clone(); + let step_name = + get_step_name_from_flow(ctx.summary.as_deref(), ctx.job.flow_step_id.as_deref()); + let content = content.to_string(); + + // Spawn task because we do not need to wait for the result + tokio::spawn(async move { + if let Err(e) = add_message_to_conversation( + &db_clone, + &memory_id, + tool_job_id, + &content, + MessageType::Tool, + &step_name, + success, + ) + .await + { + tracing::warn!( + "Failed to add tool message to conversation {}: {}", + memory_id, + e + ); + } + }); + } + } +} diff --git a/backend/windmill-worker/src/ai/types.rs b/backend/windmill-worker/src/ai/types.rs new file mode 100644 index 0000000000..457eb3b215 --- /dev/null +++ b/backend/windmill-worker/src/ai/types.rs @@ -0,0 +1,396 @@ +use crate::ai::providers::openai::OpenAIToolCall; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use std::collections::HashMap; +use windmill_common::mcp_client::McpToolSource; +use windmill_common::{ + ai_providers::AIProvider, db::DB, error::Error, flow_status::AgentAction, flows::FlowModule, + s3_helpers::S3Object, +}; +use windmill_parser::Typ; + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ContentPart { + Text { + text: String, + }, + #[serde(rename = "image_url")] + ImageUrl { + image_url: ImageUrlData, + }, + #[serde(rename = "s3_object")] + S3Object { + s3_object: S3Object, + }, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ImageUrlData { + pub url: String, // data:image/png;base64,... or https://... +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(untagged)] +pub enum OpenAIContent { + Text(String), + Parts(Vec), +} + +#[derive(Deserialize, Serialize, Clone, Default, Debug)] +pub struct OpenAIMessage { + pub role: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_call_id: Option, + #[serde(skip_serializing)] + pub agent_action: Option, +} + +/// same as OpenAIMessage but with agent_action field included in the serialization +#[derive(Serialize)] +pub struct Message<'a> { + #[serde(flatten)] + pub message: &'a OpenAIMessage, + #[serde(skip_serializing_if = "Option::is_none")] + pub agent_action: Option<&'a AgentAction>, +} + +#[derive(Serialize, Clone, Debug)] +pub struct ResponseFormat { + pub r#type: String, + pub json_schema: JsonSchemaFormat, +} + +#[derive(Serialize, Clone, Debug)] +pub struct JsonSchemaFormat { + pub name: String, + pub schema: OpenAPISchema, + #[serde(skip_serializing_if = "Option::is_none")] + pub strict: Option, +} + +#[derive(Serialize, Clone, Debug)] +pub struct ToolDefFunction { + pub name: String, + pub description: Option, + pub parameters: Box, +} + +#[derive(Serialize, Clone, Debug)] +pub struct ToolDef { + pub r#type: String, + pub function: ToolDefFunction, +} + +#[derive(Serialize, Clone, Debug)] +pub struct Tool { + pub module: Option, + pub def: ToolDef, + pub mcp_source: Option, +} + +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum OutputType { + Text, + Image, +} + +impl Default for OutputType { + fn default() -> Self { + OutputType::Text + } +} + +#[derive(Deserialize, Debug)] +pub struct AIAgentArgs { + pub provider: ProviderWithResource, + pub system_prompt: Option, + pub user_message: String, + pub temperature: Option, + pub max_completion_tokens: Option, + pub output_schema: Option, + pub output_type: Option, + pub user_images: Option>, + pub streaming: Option, + pub messages_context_length: Option, +} + +#[derive(Deserialize, Debug)] +pub struct ProviderResource { + #[serde(alias = "apiKey")] + pub api_key: String, + #[serde(alias = "baseUrl")] + pub base_url: Option, +} + +#[derive(Deserialize, Debug)] +pub struct ProviderWithResource { + pub kind: AIProvider, + pub resource: ProviderResource, + pub model: String, +} + +impl ProviderWithResource { + pub fn get_api_key(&self) -> &str { + &self.resource.api_key + } + + pub fn get_model(&self) -> &str { + &self.model + } + + pub async fn get_base_url(&self, db: &DB) -> Result { + self.kind + .get_base_url(self.resource.base_url.clone(), db) + .await + } +} + +#[derive(Serialize)] +pub struct AIAgentResult<'a> { + pub output: Box, + pub messages: Vec>, + #[serde(skip_serializing_if = "Option::is_none")] + pub wm_stream: Option, +} + +/// Events for streaming AI responses +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum StreamingEvent { + /// Individual token from the AI response + TokenDelta { content: String }, + /// Tool call has started + ToolCall { call_id: String, function_name: String }, + /// Tool call arguments are complete + ToolCallArguments { call_id: String, function_name: String, arguments: String }, + /// Tool execution has started + ToolExecution { call_id: String, function_name: String }, + /// Tool execution result + ToolResult { call_id: String, function_name: String, result: String, success: bool }, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(untagged)] +pub enum SchemaType { + Single(String), + Multiple(Vec), +} + +impl Default for SchemaType { + fn default() -> Self { + SchemaType::Single("object".to_string()) + } +} + +#[derive(Serialize, Deserialize, Default, Clone, Debug)] +pub struct OpenAPISchema { + #[serde(skip_serializing_if = "Option::is_none")] + pub r#type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub items: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub properties: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub required: Option>, + #[serde(skip_serializing_if = "Option::is_none", rename = "oneOf")] + pub one_of: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub r#enum: Option>, + #[serde( + skip_serializing_if = "Option::is_none", + rename = "additionalProperties" + )] + pub additional_properties: Option, +} + +impl OpenAPISchema { + pub fn from_str(typ: &str) -> Self { + OpenAPISchema { r#type: Some(SchemaType::Single(typ.to_string())), ..Default::default() } + } + + pub fn from_str_with_enum(typ: &str, enu: &Option>) -> Self { + OpenAPISchema { + r#type: Some(SchemaType::Single(typ.to_string())), + r#enum: enu.clone(), + ..Default::default() + } + } + + pub fn datetime() -> Self { + Self { + r#type: Some(SchemaType::Single("string".to_string())), + format: Some("date-time".to_string()), + ..Default::default() + } + } + + pub fn from_typ(typ: &Typ) -> Self { + match typ { + Typ::Str(enu) => Self::from_str_with_enum("string", enu), + Typ::Int => Self::from_str("integer"), + Typ::Float => Self::from_str("number"), + Typ::Bool => Self::from_str("boolean"), + Typ::Bytes => Self::from_str("string"), + Typ::Datetime => Self::datetime(), + Typ::Resource(_) => Self::from_str("string"), + Typ::Email => Self::from_str("string"), + Typ::Sql => Self::from_str("string"), + Typ::DynSelect(_) => Self::from_str("string"), + Typ::DynMultiselect(_) => Self::from_str("string"), + Typ::List(typ) => OpenAPISchema { + r#type: Some(SchemaType::Single("array".to_string())), + items: Some(Box::new(Self::from_typ(typ))), + ..Default::default() + }, + Typ::Object(typ) => OpenAPISchema { + r#type: Some(SchemaType::Single("object".to_string())), + items: None, + properties: typ.props.as_ref().map(|props| { + props + .iter() + .map(|prop| (prop.key.clone(), Box::new(Self::from_typ(&prop.typ)))) + .collect() + }), + required: typ + .props + .as_ref() + .map(|props| props.iter().map(|prop| prop.key.clone()).collect()), + ..Default::default() + }, + Typ::OneOf(variants) => OpenAPISchema { + r#type: Some(SchemaType::Single("object".to_string())), + one_of: Some( + variants + .iter() + .map(|variant| { + let schema = OpenAPISchema { + r#type: Some(SchemaType::Single("object".to_string())), + properties: Some( + variant + .properties + .iter() + .map(|prop| { + ( + prop.key.clone(), + Box::new( + if prop.key == "label" || prop.key == "kind" { + Self::from_str_with_enum( + "string", + &Some(vec![variant.label.clone()]), + ) + } else { + Self::from_typ(&prop.typ) + }, + ), + ) + }) + .collect(), + ), + required: Some( + variant + .properties + .iter() + .map(|prop| prop.key.clone()) + .collect(), + ), + ..Default::default() + }; + Box::new(schema) + }) + .collect(), + ), + ..Default::default() + }, + Typ::Unknown => Self::from_str("object"), + } + } + + /// Makes this schema compatible with OpenAI's strict mode by: + /// - Adding additionalProperties: false to all object types + /// - Making non-required properties nullable + /// - Ensuring all properties are in the required array + pub fn make_strict(mut self) -> Self { + // Handle this schema if it's an object type + if let Some(SchemaType::Single(ref type_str)) = self.r#type { + if type_str == "object" { + // Set additionalProperties to false + self.additional_properties = Some(false); + + if let Some(properties) = self.properties.as_mut() { + // Get original required fields + let original_required = self.required.as_ref(); + + if let Some(required) = original_required { + // Update properties to make non-required fields nullable + for (key, prop) in properties.iter_mut() { + let mut new_prop = (**prop).clone(); + // Make non-required fields nullable + if !required.contains(key) { + new_prop = new_prop.make_nullable(); + } + // Recursively make nested schemas strict + new_prop = new_prop.make_strict(); + *prop = Box::new(new_prop); + } + } + + // All properties must be in required array for strict mode + self.required = Some(properties.keys().cloned().collect()); + } + } + } + + // Recursively process nested schemas + if let Some(ref mut items) = self.items { + **items = items.as_ref().clone().make_strict(); + } + + if let Some(ref mut one_of) = self.one_of { + *one_of = one_of + .iter() + .map(|schema| Box::new(schema.as_ref().clone().make_strict())) + .collect(); + } + + self + } + + /// Makes this property nullable by converting its type to a union with null + pub fn make_nullable(mut self) -> Self { + match self.r#type.take() { + Some(SchemaType::Single(type_str)) => { + if type_str != "null" { + self.r#type = Some(SchemaType::Multiple(vec![type_str, "null".into()])); + } else { + self.r#type = Some(SchemaType::Single("null".into())); + } + } + Some(SchemaType::Multiple(mut types)) => { + if !types.iter().any(|t| t == "null") { + types.push("null".into()); + } + self.r#type = Some(SchemaType::Multiple(types)); + } + None => { + self.r#type = Some(SchemaType::Single("null".into())); + } + } + self + } +} + +/// Wrapper for S3Object with type discriminator for conversation storage +#[derive(Serialize)] +pub struct S3ObjectWithType { + #[serde(flatten)] + pub s3_object: S3Object, + pub r#type: String, +} diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs new file mode 100644 index 0000000000..fb27c419ba --- /dev/null +++ b/backend/windmill-worker/src/ai/utils.rs @@ -0,0 +1,559 @@ +use crate::ai::types::{ToolDef, ToolDefFunction}; +use anyhow::Context; +use serde_json::value::RawValue; +use sqlx::types::Json; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; +use uuid::Uuid; +use windmill_common::{ + db::DB, + error::Error, + flow_conversations::{add_message_to_conversation_tx, MessageType}, + flow_status::AgentAction, + flows::{InputTransform, Step}, + jobs::JobKind, + scripts::{ScriptHash, ScriptLang}, + worker::to_raw_value, +}; +use windmill_common::{ + flows::FlowModuleValue, + mcp_client::{McpClient, McpResource, McpToolSource}, +}; +use windmill_queue::{flow_status::get_step_of_flow_status, MiniPulledJob}; + +use crate::{ai::types::*, parse_sig_of_lang}; + +pub fn parse_raw_script_schema( + content: &str, + language: &ScriptLang, +) -> Result, Error> { + let main_arg_signature = parse_sig_of_lang(content, Some(&language), None)?.unwrap(); // safe to unwrap as langauge is some + + let schema = OpenAPISchema { + r#type: Some(SchemaType::default()), + properties: Some( + main_arg_signature + .args + .iter() + .map(|arg| { + let name = arg.name.clone(); + let typ = OpenAPISchema::from_typ(&arg.typ); + (name, Box::new(typ)) + }) + .collect(), + ), + required: Some( + main_arg_signature + .args + .iter() + .map(|arg| arg.name.clone()) + .collect(), + ), + ..Default::default() + }; + + Ok(to_raw_value(&schema)) +} + +/// Filters out properties from a JSON schema that have completed input transforms. +/// This allows AI agents to only see and fill parameters that don't have user-configured values. +pub fn filter_schema_by_input_transforms( + schema: Box, + input_transforms: &HashMap, +) -> Result, Error> { + // Parse the schema JSON + let mut schema_value: serde_json::Value = serde_json::from_str(schema.get()) + .context("Failed to parse schema JSON") + .map_err(|e| Error::ExecutionErr(e.to_string()))?; + + // Collect keys to remove (parameters with completed input transforms) + let keys_to_remove: HashSet = input_transforms + .iter() + .filter_map(|(key, transform)| { + let is_completed = match transform { + InputTransform::Static { value } => { + let val = value.get().trim(); + !val.is_empty() && val != "null" + } + InputTransform::Javascript { expr } => !expr.trim().is_empty(), + }; + if is_completed { + Some(key.clone()) + } else { + None + } + }) + .collect(); + + if !keys_to_remove.is_empty() { + // Remove completed parameters from properties + if let Some(properties) = schema_value + .get_mut("properties") + .and_then(|p| p.as_object_mut()) + { + for key in &keys_to_remove { + properties.remove(key); + } + } + + // Also remove from required array + if let Some(required) = schema_value + .get_mut("required") + .and_then(|r| r.as_array_mut()) + { + required.retain(|item| { + if let Some(key) = item.as_str() { + !keys_to_remove.contains(key) + } else { + true + } + }); + } + } + + // Convert back to RawValue + Ok(to_raw_value(&schema_value)) +} + +pub struct FlowJobRunnableIdAndRawFlow { + pub runnable_id: Option, + pub raw_flow: Option>>, + pub kind: JobKind, +} + +pub async fn get_flow_job_runnable_and_raw_flow( + db: &DB, + job_id: &uuid::Uuid, +) -> windmill_common::error::Result { + let job = sqlx::query_as!( + FlowJobRunnableIdAndRawFlow, + "SELECT runnable_id as \"runnable_id: ScriptHash\", raw_flow as \"raw_flow: _\", kind as \"kind: _\" FROM v2_job WHERE id = $1", + job_id + ) + .fetch_one(db) + .await?; + Ok(job) +} + +#[derive(Debug, Clone, Default)] +pub struct FlowContext { + pub flow_inputs: Option>>, + pub flow_status: Option, +} + +/// Get flow context (chat settings + args + flow_status) from root flow's job data +pub async fn get_flow_context(db: &DB, job: &MiniPulledJob) -> FlowContext { + let root_job_id = job + .root_job + .or(job.flow_innermost_root_job) + .or(job.parent_job); + + let Some(root_job_id) = root_job_id else { + return FlowContext::default(); + }; + + match sqlx::query!( + r#" + SELECT + j.args as "args: Json>>", + js.flow_status as "flow_status: Json" + FROM v2_job_status js + INNER JOIN v2_job j ON j.id = js.id + WHERE js.id = $1 + "#, + root_job_id + ) + .fetch_optional(db) + .await + { + Ok(Some(row)) => FlowContext { + flow_inputs: row.args.map(|j| j.0), + flow_status: row.flow_status.map(|j| j.0), + }, + Ok(None) => { + tracing::warn!( + "No flow context found for root job {} (agent job {}), returning default", + root_job_id, + job.id + ); + FlowContext::default() + } + Err(e) => { + tracing::error!("Failed to get flow context for job {}: {}", job.id, e); + FlowContext::default() + } + } +} + +// Add message to conversation +pub async fn add_message_to_conversation( + db: &DB, + conversation_id: &Uuid, + job_id: Option, + message_content: &str, + message_type: MessageType, + step_name: &Option, + success: bool, +) -> Result<(), Error> { + let mut tx = db.begin().await?; + add_message_to_conversation_tx( + &mut tx, + *conversation_id, + job_id, + &message_content, + message_type, + step_name.as_deref(), + success, + ) + .await?; + tx.commit().await?; + Ok(()) +} + +/// Find a unique tool name for structured output tool to avoid collisions with user-provided tools +pub fn find_unique_tool_name(base_name: &str, existing_tools: Option<&[ToolDef]>) -> String { + let Some(tools) = existing_tools else { + return base_name.to_string(); + }; + + if !tools.iter().any(|t| t.function.name == base_name) { + return base_name.to_string(); + } + + for i in 1..100 { + let candidate = format!("{}_{}", base_name, i); + if !tools.iter().any(|t| t.function.name == candidate) { + return candidate; + } + } + + // Fallback with process id if somehow we can't find a unique name + format!("{}_{}_fallback", base_name, std::process::id()) +} + +pub async fn update_flow_status_module_with_actions( + db: &DB, + parent_job: &Uuid, + actions: &[AgentAction], +) -> Result<(), Error> { + let step = get_step_of_flow_status(db, parent_job.to_owned()).await?; + match step { + Step::Step { idx: step, .. } => { + sqlx::query!( + r#" + UPDATE v2_job_status SET + flow_status = jsonb_set( + flow_status, + array['modules', $3::TEXT, 'agent_actions'], + $2 + ) + WHERE id = $1 + "#, + parent_job, + sqlx::types::Json(actions) as _, + step as i32 + ) + .execute(db) + .await?; + } + _ => {} + } + Ok(()) +} + +pub async fn update_flow_status_module_with_actions_success( + db: &DB, + parent_job: &Uuid, + action_success: bool, +) -> Result<(), Error> { + let step = get_step_of_flow_status(db, parent_job.to_owned()).await?; + match step { + Step::Step { idx: step, .. } => { + // Append the new bool to the existing array, or create a new array if it doesn't exist + sqlx::query!( + r#" + UPDATE v2_job_status SET + flow_status = jsonb_set( + flow_status, + array['modules', $2::TEXT, 'agent_actions_success'], + COALESCE( + flow_status->'modules'->$2->'agent_actions_success', + to_jsonb(ARRAY[]::bool[]) + ) || to_jsonb(ARRAY[$3::bool]) + ) + WHERE id = $1 + "#, + parent_job, + step as i32, + action_success + ) + .execute(db) + .await?; + } + _ => {} + } + Ok(()) +} + +/// Get step name from the flow module (summary if exists, else id) +pub fn get_step_name_from_flow( + summary: Option<&str>, + flow_step_id: Option<&str>, +) -> Option { + let flow_step_id = flow_step_id?; + Some( + summary + .map(|s| s.to_string()) + .unwrap_or_else(|| format!("AI Agent Step {}", flow_step_id)), + ) +} + +/// Claude models starts with claude if provider is anthropic, or anthropic for openrouter and other providers +pub fn is_claude_model(model: &str) -> bool { + model.starts_with("claude") || model.starts_with("anthropic") +} + +/// Cleanup MCP clients by gracefully shutting down connections +pub async fn cleanup_mcp_clients(mcp_clients: HashMap>) { + if mcp_clients.is_empty() { + return; + } + + tracing::debug!("Cleaning up {} MCP client(s)", mcp_clients.len()); + + for (resource_name, client) in mcp_clients { + // Try to unwrap the Arc to get the McpClient + match Arc::try_unwrap(client) { + Ok(client) => { + tracing::debug!("Shutting down MCP client for {}", resource_name); + if let Err(e) = client.shutdown().await { + tracing::warn!("Failed to shutdown MCP client for {}: {}", resource_name, e); + } + } + Err(arc) => { + // Other references still exist (shouldn't happen in normal flow) + tracing::warn!( + "MCP client for {} still has {} references, dropping without graceful shutdown", + resource_name, + Arc::strong_count(&arc) + ); + } + } + } +} + +/// Convert raw MCP tools to Windmill Tool format with source tracking +fn convert_mcp_tools_to_windmill_tools( + mcp_tools: &[rmcp::model::Tool], + resource_name: &str, + resource_path: &str, +) -> Result, Error> { + mcp_tools + .iter() + .map(|mcp_tool| { + let tool_name = format!("mcp_{}_{}", resource_name, mcp_tool.name); + + let mut schema_value = serde_json::to_value(&*mcp_tool.input_schema) + .context("Failed to convert MCP schema to JSON value")?; + McpClient::fix_array_schemas(&mut schema_value); + let parameters = to_raw_value(&schema_value); + + // Build the description from title and description + let description = if let Some(title) = &mcp_tool.title { + if let Some(desc) = &mcp_tool.description { + Some(format!("{}: {}", title, desc)) + } else { + Some(title.to_string()) + } + } else { + mcp_tool.description.as_ref().map(|d| d.to_string()) + }; + + let tool_def_function = + ToolDefFunction { name: tool_name.clone(), description, parameters }; + + let tool_def = ToolDef { r#type: "function".to_string(), function: tool_def_function }; + + Ok(Tool { + def: tool_def, + module: None, + mcp_source: Some(McpToolSource { + name: resource_name.to_string(), + tool_name: mcp_tool.name.to_string(), + resource_path: resource_path.to_string(), + }), + }) + }) + .collect() +} + +/// Configuration for loading tools from an MCP server resource +#[derive(Debug, Clone)] +pub struct McpResourceConfig { + pub resource_path: String, + pub include_tools: Option>, + pub exclude_tools: Option>, +} + +/// Apply include/exclude filters to a list of tools +/// Priority: include_tools > exclude_tools > all +/// - If include_tools is Some and non-empty: whitelist approach (keep only listed tools) +/// - Else if exclude_tools is Some and non-empty: blacklist approach (remove listed tools) +/// - Otherwise: no filtering (keep all tools) +fn apply_tool_filters( + tools: Vec, + include_tools: &Option>, + exclude_tools: &Option>, +) -> Vec { + // If include_tools is specified and non-empty, use whitelist approach + if let Some(include_list) = include_tools { + if !include_list.is_empty() { + return tools + .into_iter() + .filter(|tool| { + tool.mcp_source + .as_ref() + .map(|src| include_list.contains(&src.tool_name)) + .unwrap_or(false) + }) + .collect(); + } + } + + // If exclude_tools is specified and non-empty, use blacklist approach + if let Some(exclude_list) = exclude_tools { + if !exclude_list.is_empty() { + return tools + .into_iter() + .filter(|tool| { + tool.mcp_source + .as_ref() + .map(|src| !exclude_list.contains(&src.tool_name)) + .unwrap_or(true) + }) + .collect(); + } + } + + // No filtering - return all tools + tools +} + +/// Load tools from MCP servers and return both the clients and tools +/// Returns a map of resource name -> client, and a vector of tools +pub async fn load_mcp_tools( + db: &DB, + workspace_id: &str, + mcp_configs: Vec, +) -> Result<(HashMap>, Vec), Error> { + let mut all_mcp_tools = Vec::new(); + let mut mcp_clients = HashMap::new(); + + for config in mcp_configs { + tracing::debug!("Loading MCP tools from resource: {}", config.resource_path); + + let path = config.resource_path.trim_start_matches("$res:"); + let mcp_resource = { + // Fetch the resource from database + let resource= sqlx::query_scalar!( + "SELECT value as \"value: sqlx::types::Json>\" FROM resource WHERE path = $1 AND workspace_id = $2", + &path, + &workspace_id + ) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("Could not find the resource {}, update the resource path in the workspace settings", config.resource_path)))? + .ok_or_else(|| Error::BadRequest(format!("Empty resource value for {}", config.resource_path)))?; + + serde_json::from_str::(resource.0.get()) + .context("Failed to parse MCP resource")? + }; + + let resource_name = mcp_resource.name.clone(); + + // Create new MCP client for this execution + tracing::debug!("Creating fresh MCP client for {}", resource_name); + let client = McpClient::from_resource(mcp_resource, db, workspace_id) + .await + .context("Failed to create MCP client")?; + + // Get raw MCP tools from client + let raw_mcp_tools = client.available_tools(); + + // Convert to Windmill Tool format + let converted_tools = + convert_mcp_tools_to_windmill_tools(raw_mcp_tools, &resource_name, &path)?; + + // Apply include/exclude filters + let filtered_tools = apply_tool_filters( + converted_tools, + &config.include_tools, + &config.exclude_tools, + ); + + tracing::info!( + "Loaded {} tools from MCP server '{}' (filtered from {} available tools)", + filtered_tools.len(), + resource_name, + raw_mcp_tools.len() + ); + + all_mcp_tools.extend(filtered_tools); + + // Store client for later use and cleanup + let mcp_client = Arc::new(client); + mcp_clients.insert(resource_name, mcp_client); + } + + Ok((mcp_clients, all_mcp_tools)) +} + +/// Execute an MCP tool by routing the call to the appropriate MCP client +pub async fn execute_mcp_tool( + mcp_clients: &HashMap>, + mcp_source: &McpToolSource, + arguments_str: &str, +) -> Result { + // Get the MCP client from the provided map + let mcp_client = mcp_clients.get(&mcp_source.name).ok_or_else(|| { + Error::internal_err(format!( + "MCP client not found for resource: {}", + mcp_source.name + )) + })?; + + // Call the MCP tool + let result = mcp_client + .call_tool(&mcp_source.tool_name, arguments_str) + .await + .context("MCP tool call failed")?; + + Ok(result) +} + +/// Check if any tool's input transforms reference previous_result +pub fn any_tool_needs_previous_result(tools: &[Tool]) -> bool { + tools.iter().any(|tool| { + if let Some(module) = &tool.module { + if let Ok(module_value) = module.get_value() { + let input_transforms = match module_value { + FlowModuleValue::Script { input_transforms, .. } => input_transforms, + FlowModuleValue::RawScript { input_transforms, .. } => input_transforms, + FlowModuleValue::FlowScript { input_transforms, .. } => input_transforms, + _ => return false, + }; + + return input_transforms.iter().any(|(_, transform)| { + if let windmill_common::flows::InputTransform::Javascript { expr } = transform { + expr.contains("previous_result") + } else { + false + } + }); + } + } + false + }) +} diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index 9778884553..f9207e23a5 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -1,1051 +1,81 @@ +use crate::ai::tools::{execute_tool_calls, ToolExecutionContext}; +use crate::ai::utils::{ + add_message_to_conversation, any_tool_needs_previous_result, cleanup_mcp_clients, + filter_schema_by_input_transforms, find_unique_tool_name, get_flow_context, + get_flow_job_runnable_and_raw_flow, get_step_name_from_flow, is_claude_model, load_mcp_tools, + parse_raw_script_schema, update_flow_status_module_with_actions, + update_flow_status_module_with_actions_success, +}; +use crate::memory_oss::{read_from_memory, write_to_memory}; +use crate::worker_flow::{get_previous_job_result, get_transform_context}; use async_recursion::async_recursion; use regex::Regex; -use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::{collections::HashMap, sync::Arc}; -#[cfg(feature = "benchmark")] -use windmill_common::bench::BenchmarkIter; +use uuid::Uuid; +use windmill_common::mcp_client::McpClient; use windmill_common::{ - ai_providers::AIProvider, - auth::get_job_perms, + ai_providers::AZURE_API_VERSION, cache, client::AuthedClient, db::DB, - error::{self, to_anyhow, Error}, + error::{self, Error}, + flow_conversations::MessageType, flow_status::AgentAction, - flows::{FlowModule, FlowModuleValue, Step}, + flows::{FlowModule, FlowModuleValue, ToolValue}, get_latest_hash_for_path, jobs::JobKind, - scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, + scripts::get_full_hub_script_by_path, utils::{StripPath, HTTP_CLIENT}, worker::{to_raw_value, Connection}, }; -use windmill_parser::Typ; -use windmill_queue::{ - flow_status::get_step_of_flow_status, get_mini_pulled_job, push, CanceledBy, JobCompleted, - MiniPulledJob, PushArgs, PushIsolationLevel, -}; +use windmill_queue::{CanceledBy, MiniPulledJob}; use crate::{ - common::{build_args_map, error_to_value, OccupancyMetrics}, - create_job_dir, + ai::{ + image_handler::upload_image_to_s3, + query_builder::{ + create_query_builder, BuildRequestArgs, ParsedResponse, StreamEventProcessor, + }, + types::*, + }, + common::{build_args_map, resolve_job_timeout, OccupancyMetrics, StreamNotifier}, handle_child::run_future_with_polling_update_job_poller, - handle_queued_job, parse_sig_of_lang, - result_processor::handle_non_flow_job_error, - worker_flow::{raw_script_to_payload, script_to_payload}, - JobCompletedSender, SendResult, SendResultPayload, + JobCompletedSender, }; -const MAX_AGENT_ITERATIONS: usize = 10; - lazy_static::lazy_static! { static ref TOOL_NAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]+$").unwrap(); -} -/// Find a unique tool name to avoid collisions with user-provided tools -fn find_unique_tool_name(base_name: &str, existing_tools: Option<&[ToolDef]>) -> String { - let Some(tools) = existing_tools else { - return base_name.to_string(); - }; - - if !tools.iter().any(|t| t.function.name == base_name) { - return base_name.to_string(); - } - - for i in 1..100 { - let candidate = format!("{}_{}", base_name, i); - if !tools.iter().any(|t| t.function.name == candidate) { - return candidate; - } - } - - // Fallback with process id if somehow we can't find a unique name - format!("{}_{}_fallback", base_name, std::process::id()) -} - -#[derive(Deserialize, Serialize, Clone, Debug)] -struct OpenAIFunction { - name: String, - arguments: String, -} - -#[derive(Deserialize, Serialize, Clone, Debug)] -struct OpenAIToolCall { - id: String, - function: OpenAIFunction, - r#type: String, -} - -#[derive(Deserialize, Serialize, Clone, Default)] -struct OpenAIMessage { - role: String, - #[serde(skip_serializing_if = "Option::is_none")] - content: Option, - #[serde(skip_serializing_if = "Option::is_none")] - tool_calls: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - tool_call_id: Option, - #[serde(skip_serializing)] - agent_action: Option, -} - -/// same as OpenAIMessage but with agent_action field included in the serialization -#[derive(Serialize)] -struct Message<'a> { - #[serde(flatten)] - message: &'a OpenAIMessage, - #[serde(skip_serializing_if = "Option::is_none")] - agent_action: Option<&'a AgentAction>, -} - -#[derive(Deserialize)] -struct OpenAIChoice { - message: OpenAIMessage, -} - -#[derive(Deserialize)] -struct OpenAIResponse { - choices: Vec, -} - -#[derive(Serialize)] -struct OpenAIRequest<'a> { - model: &'a str, - messages: &'a Vec, - #[serde(skip_serializing_if = "Option::is_none")] - tools: Option<&'a Vec>, - #[serde(skip_serializing_if = "Option::is_none")] - temperature: Option, - #[serde(skip_serializing_if = "Option::is_none")] - max_completion_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - response_format: Option, -} - -#[derive(Serialize, Clone, Debug)] -struct ResponseFormat { - r#type: String, - json_schema: JsonSchemaFormat, -} - -#[derive(Serialize, Clone, Debug)] -struct JsonSchemaFormat { - name: String, - schema: OpenAPISchema, - #[serde(skip_serializing_if = "Option::is_none")] - strict: Option, -} - -#[derive(Serialize, Clone, Debug)] -struct ToolDefFunction { - name: String, - description: Option, - parameters: Box, -} - -#[derive(Serialize, Clone, Debug)] -struct ToolDef { - r#type: String, - function: ToolDefFunction, -} - -struct Tool { - module: FlowModule, - def: ToolDef, -} - -#[derive(Deserialize, Debug)] -struct AIAgentArgs { - provider: ProviderWithResource, - system_prompt: Option, - user_message: String, - temperature: Option, - max_completion_tokens: Option, - output_schema: Option, -} - -#[derive(Deserialize, Debug)] -struct ProviderResource { - #[serde(alias = "apiKey")] - api_key: String, - #[serde(alias = "baseUrl")] - base_url: Option, -} - -#[derive(Deserialize, Debug)] -struct ProviderWithResource { - kind: AIProvider, - resource: ProviderResource, - model: String, -} - -impl ProviderWithResource { - fn get_api_key(&self) -> &str { - &self.resource.api_key - } - - fn get_model(&self) -> &str { - &self.model - } - - async fn get_base_url(&self, db: &DB) -> Result { - self.kind - .get_base_url(self.resource.base_url.clone(), db) - .await - } -} - -#[derive(Serialize)] -struct AIAgentResult<'a> { - output: Box, - messages: Vec>, -} - -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(untagged)] -enum SchemaType { - Single(String), - Multiple(Vec), -} - -impl Default for SchemaType { - fn default() -> Self { - SchemaType::Single("object".to_string()) - } -} - -#[derive(Serialize, Deserialize, Default, Clone, Debug)] -struct OpenAPISchema { - #[serde(skip_serializing_if = "Option::is_none")] - r#type: Option, - #[serde(skip_serializing_if = "Option::is_none")] - items: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - properties: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - required: Option>, - #[serde(skip_serializing_if = "Option::is_none", rename = "oneOf")] - one_of: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - format: Option, - #[serde(skip_serializing_if = "Option::is_none")] - r#enum: Option>, - #[serde( - skip_serializing_if = "Option::is_none", - rename = "additionalProperties" - )] - additional_properties: Option, -} - -impl OpenAPISchema { - fn from_str(typ: &str) -> Self { - OpenAPISchema { r#type: Some(SchemaType::Single(typ.to_string())), ..Default::default() } - } - - fn from_str_with_enum(typ: &str, enu: &Option>) -> Self { - OpenAPISchema { - r#type: Some(SchemaType::Single(typ.to_string())), - r#enum: enu.clone(), - ..Default::default() - } - } - - fn datetime() -> Self { - Self { - r#type: Some(SchemaType::Single("string".to_string())), - format: Some("date-time".to_string()), - ..Default::default() - } - } - - fn from_typ(typ: &Typ) -> Self { - match typ { - Typ::Str(enu) => Self::from_str_with_enum("string", enu), - Typ::Int => Self::from_str("integer"), - Typ::Float => Self::from_str("number"), - Typ::Bool => Self::from_str("boolean"), - Typ::Bytes => Self::from_str("string"), - Typ::Datetime => Self::datetime(), - Typ::Resource(_) => Self::from_str("string"), - Typ::Email => Self::from_str("string"), - Typ::Sql => Self::from_str("string"), - Typ::DynSelect(_) => Self::from_str("string"), - Typ::DynMultiselect(_) => Self::from_str("string"), - Typ::List(typ) => OpenAPISchema { - r#type: Some(SchemaType::Single("array".to_string())), - items: Some(Box::new(Self::from_typ(typ))), - ..Default::default() - }, - Typ::Object(typ) => OpenAPISchema { - r#type: Some(SchemaType::Single("object".to_string())), - items: None, - properties: typ.props.as_ref().map(|props| { - props - .iter() - .map(|prop| (prop.key.clone(), Box::new(Self::from_typ(&prop.typ)))) - .collect() - }), - required: typ - .props - .as_ref() - .map(|props| props.iter().map(|prop| prop.key.clone()).collect()), - ..Default::default() - }, - Typ::OneOf(variants) => OpenAPISchema { - r#type: Some(SchemaType::Single("object".to_string())), - one_of: Some( - variants - .iter() - .map(|variant| { - let schema = OpenAPISchema { - r#type: Some(SchemaType::Single("object".to_string())), - properties: Some( - variant - .properties - .iter() - .map(|prop| { - ( - prop.key.clone(), - Box::new( - if prop.key == "label" || prop.key == "kind" { - Self::from_str_with_enum( - "string", - &Some(vec![variant.label.clone()]), - ) - } else { - Self::from_typ(&prop.typ) - }, - ), - ) - }) - .collect(), - ), - required: Some( - variant - .properties - .iter() - .map(|prop| prop.key.clone()) - .collect(), - ), - ..Default::default() - }; - Box::new(schema) - }) - .collect(), - ), - ..Default::default() - }, - Typ::Unknown => Self::from_str("object"), - } - } - - /// Makes this schema compatible with OpenAI's strict mode by: - /// - Adding additionalProperties: false to all object types - /// - Making non-required properties nullable - /// - Ensuring all properties are in the required array - fn make_strict(mut self) -> Self { - // Handle this schema if it's an object type - if let Some(SchemaType::Single(ref type_str)) = self.r#type { - if type_str == "object" { - // Set additionalProperties to false - self.additional_properties = Some(false); - - if let Some(properties) = self.properties.as_mut() { - // Get original required fields - let original_required = self.required.as_ref(); - - if let Some(required) = original_required { - // Update properties to make non-required fields nullable - for (key, prop) in properties.iter_mut() { - let mut new_prop = (**prop).clone(); - // Make non-required fields nullable - if !required.contains(key) { - new_prop = new_prop.make_nullable(); + /// Parse AI_HTTP_HEADERS environment variable into a vector of (header_name, header_value) tuples + /// Format: "header1: value1, header2: value2" + static ref AI_HTTP_HEADERS: Vec<(String, String)> = { + std::env::var("AI_HTTP_HEADERS") + .ok() + .map(|headers_str| { + headers_str + .split(',') + .filter_map(|header| { + let parts: Vec<&str> = header.splitn(2, ':').collect(); + if parts.len() == 2 { + let name = parts[0].trim().to_string(); + let value = parts[1].trim().to_string(); + if !name.is_empty() && !value.is_empty() { + Some((name, value)) + } else { + None } - // Recursively make nested schemas strict - new_prop = new_prop.make_strict(); - *prop = Box::new(new_prop); + } else { + None } - } - - // All properties must be in required array for strict mode - self.required = Some(properties.keys().cloned().collect()); - } - } - } - - // Recursively process nested schemas - if let Some(ref mut items) = self.items { - **items = items.as_ref().clone().make_strict(); - } - - if let Some(ref mut one_of) = self.one_of { - *one_of = one_of - .iter() - .map(|schema| Box::new(schema.as_ref().clone().make_strict())) - .collect(); - } - - self - } - - /// Makes this property nullable by converting its type to a union with null - fn make_nullable(mut self) -> Self { - match self.r#type.take() { - Some(SchemaType::Single(type_str)) => { - if type_str != "null" { - self.r#type = Some(SchemaType::Multiple(vec![type_str, "null".into()])); - } else { - self.r#type = Some(SchemaType::Single("null".into())); - } - } - Some(SchemaType::Multiple(mut types)) => { - if !types.iter().any(|t| t == "null") { - types.push("null".into()); - } - self.r#type = Some(SchemaType::Multiple(types)); - } - None => { - self.r#type = Some(SchemaType::Single("null".into())); - } - } - self - } -} - -async fn update_flow_status_module_with_actions( - db: &DB, - parent_job: &uuid::Uuid, - actions: &[AgentAction], -) -> Result<(), Error> { - let step = get_step_of_flow_status(db, parent_job.to_owned()).await?; - match step { - Step::Step(step) => { - sqlx::query!( - r#" - UPDATE v2_job_status SET - flow_status = jsonb_set( - flow_status, - array['modules', $3::TEXT, 'agent_actions'], - $2 - ) - WHERE id = $1 - "#, - parent_job, - sqlx::types::Json(actions) as _, - step as i32 - ) - .execute(db) - .await?; - } - _ => {} - } - Ok(()) -} - -async fn update_flow_status_module_with_actions_success( - db: &DB, - parent_job: &uuid::Uuid, - action_success: bool, -) -> Result<(), Error> { - let step = get_step_of_flow_status(db, parent_job.to_owned()).await?; - match step { - Step::Step(step) => { - // Append the new bool to the existing array, or create a new array if it doesn't exist - sqlx::query!( - r#" - UPDATE v2_job_status SET - flow_status = jsonb_set( - flow_status, - array['modules', $2::TEXT, 'agent_actions_success'], - COALESCE( - flow_status->'modules'->$2->'agent_actions_success', - to_jsonb(ARRAY[]::bool[]) - ) || to_jsonb(ARRAY[$3::bool]) - ) - WHERE id = $1 - "#, - parent_job, - step as i32, - action_success - ) - .execute(db) - .await?; - } - _ => {} - } - Ok(()) -} - -fn parse_raw_script_schema(content: &str, language: &ScriptLang) -> Result, Error> { - let main_arg_signature = parse_sig_of_lang(content, Some(&language), None)?.unwrap(); // safe to unwrap as langauge is some - - let schema = OpenAPISchema { - r#type: Some(SchemaType::default()), - properties: Some( - main_arg_signature - .args - .iter() - .map(|arg| { - let name = arg.name.clone(); - let typ = OpenAPISchema::from_typ(&arg.typ); - (name, Box::new(typ)) - }) - .collect(), - ), - required: Some( - main_arg_signature - .args - .iter() - .map(|arg| arg.name.clone()) - .collect(), - ), - ..Default::default() + }) + .collect() + }) + .unwrap_or_default() }; - - Ok(to_raw_value(&schema)) } -#[async_recursion] // we only need it because handle_queued_job could call this function again but in practice it won't because we only accept workspace/raw script flow modules -async fn call_tool( - // connection - db: &DB, - conn: &Connection, - - // agent job and flow step id - agent_job: &MiniPulledJob, - - // tool - tool_module: &FlowModule, - tool_call: &OpenAIToolCall, - job_id: uuid::Uuid, - - // execution context - client: &AuthedClient, - occupancy_metrics: &mut OccupancyMetrics, - base_internal_url: &str, - worker_dir: &str, - worker_name: &str, - hostname: &str, - job_completed_tx: &JobCompletedSender, - killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, -) -> error::Result<(bool, Arc>)> { - let tool_call_args = - serde_json::from_str::>>(&tool_call.function.arguments)?; - - let job_payload = match tool_module.get_value()? { - FlowModuleValue::Script { path: script_path, hash: script_hash, tag_override, .. } => { - let payload = script_to_payload( - script_hash, - script_path, - db, - agent_job, - tool_module, - tag_override, - tool_module.apply_preprocessor, - ) - .await?; - payload - } - FlowModuleValue::RawScript { - path, - content, - language, - lock, - tag, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - .. - } => { - let path = path.unwrap_or_else(|| { - format!("{}/tools/{}", agent_job.runnable_path(), tool_module.id) - }); - - let payload = raw_script_to_payload( - path, - content, - language, - lock, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - tool_module, - tag, - tool_module.delete_after_use.unwrap_or(false), - ); - payload - } - _ => { - return Err(Error::internal_err(format!( - "Unsupported tool: {}", - tool_call.function.name - ))); - } - }; - - let mut tx = db.begin().await?; - - let job_perms = get_job_perms(&mut *tx, &agent_job.id, &agent_job.workspace_id) - .await? - .map(|x| x.into()); - - let (email, permissioned_as) = if let Some(on_behalf_of) = job_payload.on_behalf_of.as_ref() { - (&on_behalf_of.email, on_behalf_of.permissioned_as.clone()) - } else { - ( - &agent_job.permissioned_as_email, - agent_job.permissioned_as.to_owned(), - ) - }; - - let job_priority = tool_module.priority.or(agent_job.priority); - - let tx = PushIsolationLevel::Transaction(tx); - let (uuid, tx) = push( - db, - tx, - &agent_job.workspace_id, - job_payload.payload, - PushArgs { args: &tool_call_args, extra: None }, - &agent_job.created_by, - email, - permissioned_as, - Some(&format!("job-span-{}", agent_job.id)), - None, - agent_job.schedule_path(), - Some(agent_job.id), - None, - None, - Some(job_id), - false, - false, - None, - agent_job.visible_to_owner, - Some(agent_job.tag.clone()), // we reuse the same tag as the agent job because it's run on the same worker - job_payload.timeout, - None, - job_priority, - job_perms.as_ref(), - true, - ) - .await?; - - tx.commit().await?; - - let tool_job = get_mini_pulled_job(db, &uuid).await?; - - let Some(tool_job) = tool_job else { - return Err(Error::internal_err("Tool job not found".to_string())); - }; - - let tool_job = Arc::new(tool_job); - - let job_dir = create_job_dir(&worker_dir, agent_job.id).await; - - let (inner_job_completed_tx, inner_job_completed_rx) = JobCompletedSender::new(&conn, 1); - - let inner_job_completed_rx = inner_job_completed_rx.expect( - "inner_job_completed_tx should be set as agent jobs are not supported on agent workers", - ); - - #[cfg(feature = "benchmark")] - let mut bench = BenchmarkIter::new(); - - match handle_queued_job( - tool_job.clone(), - None, - None, - None, - None, - conn, - client, - hostname, - worker_name, - worker_dir, - &job_dir, - None, - base_internal_url, - inner_job_completed_tx, - occupancy_metrics, - killpill_rx, - None, - #[cfg(feature = "benchmark")] - &mut bench, - ) - .await - { - Err(err) => { - let err_string = format!("{}: {}", err.name(), err.to_string()); - let err_json = error_to_value(&err); - let _ = handle_non_flow_job_error( - db, - &tool_job, - 0, - None, - err_string, - err_json, - worker_name, - ) - .await; - Err(err) - } - Ok(success) => { - let send_result = inner_job_completed_rx.bounded_rx.try_recv().ok(); - - let result = if let Some(SendResult { - result: SendResultPayload::JobCompleted(JobCompleted { result, .. }), - .. - }) = send_result.as_ref() - { - job_completed_tx - .send(send_result.as_ref().unwrap().result.clone(), true) - .await - .map_err(to_anyhow)?; - result - } else { - if let Some(send_result) = send_result { - job_completed_tx - .send(send_result.result, true) - .await - .map_err(to_anyhow)?; - } - return Err(Error::internal_err( - "Tool job completed but no result".to_string(), - )); - }; - - Ok((success, result.clone())) - } - } -} - -async fn run_agent( - // connection - db: &DB, - conn: &Connection, - - // agent job and flow data - job: &MiniPulledJob, - parent_job: &uuid::Uuid, - args: AIAgentArgs, - tools: Vec, - - // job execution context - client: &AuthedClient, - occupancy_metrics: &mut OccupancyMetrics, - job_completed_tx: &JobCompletedSender, - worker_dir: &str, - base_internal_url: &str, - worker_name: &str, - hostname: &str, - killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, -) -> error::Result> { - let mut messages = if let Some(system_prompt) = args.system_prompt.filter(|s| !s.is_empty()) { - vec![OpenAIMessage { - role: "system".to_string(), - content: Some(system_prompt), - ..Default::default() - }] - } else { - vec![] - }; - - messages.push(OpenAIMessage { - role: "user".to_string(), - content: Some(args.user_message), - ..Default::default() - }); - - let mut actions = vec![]; - - let mut content = None; - - let base_url = args.provider.get_base_url(db).await?; - let api_key = args.provider.get_api_key(); - - let mut tool_defs: Option> = if tools.is_empty() { - None - } else { - Some(tools.iter().map(|t| t.def.clone()).collect()) - }; - - let has_output_properties = args - .output_schema - .as_ref() - .and_then(|schema| schema.properties.as_ref()) - .map(|props| !props.is_empty()) - .unwrap_or(false); - let provider_is_anthropic = args.provider.kind.is_anthropic(); - let is_openrouter_anthropic = args.provider.kind == AIProvider::OpenRouter - && args.provider.model.starts_with("anthropic/"); - let is_anthropic = provider_is_anthropic || is_openrouter_anthropic; - let mut response_format: Option = None; - let mut used_structured_output_tool = false; - let mut structured_output_tool_name: Option = None; - - if has_output_properties { - let schema = args.output_schema.as_ref().unwrap(); // we know it's some because of the check above - if is_anthropic { - // if output schema is provided, and provider is anthropic, add a structured_output tool in the list of tools - let unique_tool_name = find_unique_tool_name("structured_output", tool_defs.as_deref()); - structured_output_tool_name = Some(unique_tool_name.clone()); - - let output_tool = ToolDef { - r#type: "function".to_string(), - function: ToolDefFunction { - name: unique_tool_name, - description: Some( - "This tool MUST be used last to return a structured JSON object as the final output." - .to_string(), - ), - parameters: to_raw_value(&schema), - }, - }; - if let Some(ref mut existing_tools) = tool_defs { - existing_tools.push(output_tool); - } else { - tool_defs = Some(vec![output_tool]); - } - } else { - // if output schema is provided, and provider is openai, add a response_format with json_schema - let strict_schema = schema.clone().make_strict(); - response_format = Some(ResponseFormat { - r#type: "json_schema".to_string(), - json_schema: JsonSchemaFormat { - name: "structured_output".to_string(), - schema: strict_schema, - strict: Some(true), - }, - }); - } - } - - for i in 0..MAX_AGENT_ITERATIONS { - if used_structured_output_tool { - break; - } - - let response = { - let resp = HTTP_CLIENT - .post(format!("{}/chat/completions", base_url)) - .bearer_auth(api_key) - .json(&OpenAIRequest { - model: args.provider.get_model(), - messages: &messages, - tools: tool_defs.as_ref(), - temperature: args.temperature, - max_completion_tokens: args.max_completion_tokens, - response_format: if has_output_properties && !is_anthropic { - response_format.clone() - } else { - None - }, - }) - .send() - .await - .map_err(|e| Error::internal_err(format!("Failed to call API: {}", e)))?; - - match resp.error_for_status_ref() { - Ok(_) => resp, - Err(e) => { - let status = resp.status(); - let text = resp - .text() - .await - .unwrap_or_else(|_| "".to_string()); - tracing::error!( - "Non 200 response from API: status: {}, body: {}", - status, - text - ); - return Err(Error::internal_err(format!( - "Non 200 response from API: {} - {}", - e, text - ))); - } - } - }; - - let mut response = response - .json::() - .await - .map_err(|e| Error::internal_err(format!("Failed to parse API response: {}", e)))?; - - let first_choice = response - .choices - .pop() - .ok_or_else(|| Error::internal_err("No response from API"))?; - - content = first_choice.message.content; - let tool_calls = first_choice.message.tool_calls.unwrap_or_default(); - - if let Some(ref content) = content { - actions.push(AgentAction::Message {}); - messages.push(OpenAIMessage { - role: "assistant".to_string(), - content: Some(content.clone()), - agent_action: Some(AgentAction::Message {}), - ..Default::default() - }); - - update_flow_status_module_with_actions(db, parent_job, &actions).await?; - update_flow_status_module_with_actions_success(db, parent_job, true).await?; - } - - if tool_calls.is_empty() { - break; - } else if i == MAX_AGENT_ITERATIONS - 1 { - return Err(Error::internal_err( - "AI agent reached max iterations, but there are still tool calls".to_string(), - )); - } - - messages.push(OpenAIMessage { - role: "assistant".to_string(), - tool_calls: Some(tool_calls.clone()), - ..Default::default() - }); - - for tool_call in tool_calls.iter() { - // Structured output tool is used, we stop here as this will be the final output - if structured_output_tool_name - .as_ref() - .map_or(false, |name| tool_call.function.name == *name) - { - used_structured_output_tool = true; - messages.push(OpenAIMessage { - role: "tool".to_string(), - content: Some("Successfully ran structured_output tool".to_string()), - tool_call_id: Some(tool_call.id.clone()), - ..Default::default() - }); - messages.push(OpenAIMessage { - role: "assistant".to_string(), - content: Some(tool_call.function.arguments.clone()), - agent_action: Some(AgentAction::Message {}), - ..Default::default() - }); - content = Some(tool_call.function.arguments.clone()); - break; - } - - let tool = tools - .iter() - .find(|t| t.def.function.name == tool_call.function.name); - if let Some(tool) = tool { - let job_id = ulid::Ulid::new().into(); - actions.push(AgentAction::ToolCall { - job_id, - function_name: tool_call.function.name.clone(), - module_id: tool.module.id.clone(), - }); - - update_flow_status_module_with_actions(db, parent_job, &actions).await?; - - match call_tool( - db, - conn, - job, - &tool.module, - &tool_call, - job_id, - client, - occupancy_metrics, - base_internal_url, - worker_dir, - worker_name, - hostname, - job_completed_tx, - killpill_rx, - ) - .await - { - Ok((success, result)) => { - messages.push(OpenAIMessage { - role: "tool".to_string(), - content: Some(result.get().to_string()), - tool_call_id: Some(tool_call.id.clone()), - agent_action: Some(AgentAction::ToolCall { - job_id, - function_name: tool_call.function.name.clone(), - module_id: tool.module.id.clone(), - }), - ..Default::default() - }); - update_flow_status_module_with_actions_success(db, parent_job, success) - .await?; - } - Err(err) => { - let err_string = format!("{}: {}", err.name(), err.to_string()); - messages.push(OpenAIMessage { - role: "tool".to_string(), - content: Some(format!("Error running tool: {}", err_string)), - tool_call_id: Some(tool_call.id.clone()), - agent_action: Some(AgentAction::ToolCall { - job_id, - function_name: tool_call.function.name.clone(), - module_id: tool.module.id.clone(), - }), - ..Default::default() - }); - update_flow_status_module_with_actions_success(db, parent_job, false) - .await?; - } - } - } else { - return Err(Error::internal_err(format!( - "Tool not found: {}", - tool_call.function.name - ))); - } - } - } - - let final_messages: Vec = messages - .iter() - .map(|m| Message { message: m, agent_action: m.agent_action.as_ref() }) - .collect(); - - // Parse content as JSON, fallback to string if it fails - let output_value = match content { - Some(content_str) => match has_output_properties { - true => serde_json::from_str::>(&content_str).map_err(|_e| { - Error::internal_err(format!( - "Failed to parse structured output: {}", - content_str - )) - })?, - false => to_raw_value(&content_str), - }, - None => to_raw_value(&""), - }; - - Ok(to_raw_value(&AIAgentResult { - output: output_value, - messages: final_messages, - })) -} - -pub struct FlowJobRunnableIdAndRawFlow { - pub runnable_id: Option, - pub raw_flow: Option>>, - pub kind: JobKind, -} - -pub async fn get_flow_job_runnable_and_raw_flow( - db: &DB, - job_id: &uuid::Uuid, -) -> windmill_common::error::Result { - let job = sqlx::query_as!( - FlowJobRunnableIdAndRawFlow, - "SELECT runnable_id as \"runnable_id: ScriptHash\", raw_flow as \"raw_flow: _\", kind as \"kind: _\" FROM v2_job WHERE id = $1", - job_id - ) - .fetch_one(db) - .await?; - Ok(job) -} +const MAX_AGENT_ITERATIONS: usize = 10; pub async fn handle_ai_agent_job( // connection @@ -1066,9 +96,9 @@ pub async fn handle_ai_agent_job( worker_name: &str, hostname: &str, killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, + has_stream: &mut bool, ) -> Result, Error> { let args = build_args_map(job, client, conn).await?; - let args = serde_json::from_str::(&serde_json::to_string(&args)?)?; let Some(flow_step_id) = &job.flow_step_id else { @@ -1102,6 +132,7 @@ pub async fn handle_ai_agent_job( let value = flow_data.value(); let module = value.modules.iter().find(|m| m.id == *flow_step_id); + let summary = module.as_ref().and_then(|m| m.summary.clone()); let Some(module) = module else { return Err(Error::internal_err( @@ -1115,7 +146,38 @@ pub async fn handle_ai_agent_job( )); }; - let tools = futures::future::try_join_all(tools.into_iter().map(|mut t| { + // Separate Windmill tools from MCP tools and extract MCP resource configs + let mut windmill_modules: Vec = Vec::new(); + let mut mcp_configs: Vec = Vec::new(); + + for tool in tools { + match &tool.value { + ToolValue::Mcp(mcp_config) => { + // This is an MCP tool - extract config + tracing::debug!( + "MCP server module: path={}, include={:?}, exclude={:?}", + mcp_config.resource_path, + mcp_config.include_tools, + mcp_config.exclude_tools + ); + mcp_configs.push(crate::ai::utils::McpResourceConfig { + resource_path: mcp_config.resource_path.clone(), + include_tools: Some(mcp_config.include_tools.clone()), + exclude_tools: Some(mcp_config.exclude_tools.clone()), + }); + } + ToolValue::FlowModule(_) => { + // Regular Windmill flow module (script, flow, etc.) - convert to FlowModule + tracing::debug!("Windmill module: {:?}", tool.id); + if let Some(flow_module) = Option::::from(&tool) { + windmill_modules.push(flow_module); + } + } + } + } + + // Process Windmill flow modules into Tool definitions + let tools = futures::future::try_join_all(windmill_modules.into_iter().map(|mut t| { let conn = conn; let db = db; let job = job; @@ -1127,63 +189,69 @@ pub async fn handle_ai_agent_job( ))); }; - let schema = match &t.get_value() { - Ok(FlowModuleValue::Script { + // Extract schema and input_transforms from the module value + let module_value = t.get_value()?; + let (schema, input_transforms) = match &module_value { + FlowModuleValue::Script { hash, path, tag_override, input_transforms, is_trigger, - }) => match hash { - Some(hash) => { - let (_, metadata) = cache::script::fetch(conn, hash.clone()).await?; - Ok::<_, Error>( - metadata - .schema - .clone() - .map(|s| RawValue::from_string(s).ok()) - .flatten(), - ) - } - None => { - if path.starts_with("hub/") { - let hub_script = get_full_hub_script_by_path( - StripPath(path.to_string()), - &HTTP_CLIENT, - None, + pass_flow_input_directly, + } => { + let schema = match hash { + Some(hash) => { + let (_, metadata) = cache::script::fetch(conn, hash.clone()).await?; + Ok::<_, Error>( + metadata + .schema + .clone() + .map(|s| RawValue::from_string(s).ok()) + .flatten(), ) - .await?; - Ok(Some(hub_script.schema)) - } else { - let hash = get_latest_hash_for_path(db, &job.workspace_id, path, true) + } + None => { + if path.starts_with("hub/") { + let hub_script = get_full_hub_script_by_path( + StripPath(path.to_string()), + &HTTP_CLIENT, + None, + ) + .await?; + Ok(Some(hub_script.schema)) + } else { + let hash = get_latest_hash_for_path( + db, + &job.workspace_id, + path.as_str(), + true, + ) .await? .0; - // update module definition to use a fixed hash so all tool calls match the same schema - t.value = to_raw_value(&FlowModuleValue::Script { - hash: Some(hash), - path: path.clone(), - tag_override: tag_override.clone(), - input_transforms: input_transforms.clone(), - is_trigger: *is_trigger, - }); - let (_, metadata) = cache::script::fetch(conn, hash).await?; - Ok(metadata - .schema - .clone() - .map(|s| RawValue::from_string(s).ok()) - .flatten()) + // update module definition to use a fixed hash so all tool calls match the same schema + t.value = to_raw_value(&FlowModuleValue::Script { + hash: Some(hash), + path: path.clone(), + tag_override: tag_override.clone(), + input_transforms: input_transforms.clone(), + is_trigger: *is_trigger, + pass_flow_input_directly: *pass_flow_input_directly, + }); + let (_, metadata) = cache::script::fetch(conn, hash).await?; + Ok(metadata + .schema + .clone() + .map(|s| RawValue::from_string(s).ok()) + .flatten()) + } } - } - }, - Ok(FlowModuleValue::RawScript { content, language, .. }) => { - Ok(Some(parse_raw_script_schema(&content, &language)?)) + }?; + (schema, input_transforms) } - Err(e) => { - return Err(Error::internal_err(format!( - "Invalid tool {}: {}", - summary, - e.to_string() - ))); + FlowModuleValue::RawScript { content, language, input_transforms, .. } => { + let schema = Some(parse_raw_script_schema(&content, &language)?); + (schema, input_transforms) } _ => { return Err(Error::internal_err(format!( @@ -1191,7 +259,14 @@ pub async fn handle_ai_agent_job( summary ))); } - }?; + }; + + // Filter schema based on user given input transforms + let schema = if let Some(s) = schema { + Some(filter_schema_by_input_transforms(s, input_transforms)?) + } else { + None + }; Ok(Tool { def: ToolDef { @@ -1208,21 +283,40 @@ pub async fn handle_ai_agent_job( }), }, }, - module: t, + module: Some(t), + mcp_source: None, }) } })) .await?; + // Load MCP tools if configured + let mut tools = tools; + let mcp_clients = if !mcp_configs.is_empty() { + let (clients, mcp_tools) = load_mcp_tools(db, &job.workspace_id, mcp_configs).await?; + tools.extend(mcp_tools); + clients + } else { + HashMap::new() + }; + let mut inner_occupancy_metrics = occupancy_metrics.clone(); + let stream_notifier = StreamNotifier::new(conn, job); + + if let Some(stream_notifier) = stream_notifier { + stream_notifier.update_flow_status_with_stream_job(); + } + let agent_fut = run_agent( db, conn, job, parent_job, - args, - tools, + &args, + &tools, + &mcp_clients, + summary.as_deref(), client, &mut inner_occupancy_metrics, job_completed_tx, @@ -1231,6 +325,7 @@ pub async fn handle_ai_agent_job( worker_name, hostname, killpill_rx, + has_stream, ); let result = run_future_with_polling_update_job_poller( @@ -1247,5 +342,547 @@ pub async fn handle_ai_agent_job( ) .await?; + // Cleanup MCP clients + cleanup_mcp_clients(mcp_clients).await; + Ok(result) } + +#[async_recursion] +pub async fn run_agent( + // connection + db: &DB, + conn: &Connection, + + // agent job and flow data + job: &MiniPulledJob, + parent_job: &Uuid, + args: &AIAgentArgs, + tools: &[Tool], + mcp_clients: &HashMap>, + summary: Option<&str>, + + // job execution context + client: &AuthedClient, + occupancy_metrics: &mut OccupancyMetrics, + job_completed_tx: &JobCompletedSender, + worker_dir: &str, + base_internal_url: &str, + worker_name: &str, + hostname: &str, + killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, + has_stream: &mut bool, +) -> error::Result> { + let output_type = args.output_type.as_ref().unwrap_or(&OutputType::Text); + let base_url = args.provider.get_base_url(db).await?; + let api_key = args.provider.get_api_key(); + + // Create the query builder for the provider + let query_builder = create_query_builder(&args.provider); + + // Initialize messages + let mut messages = + if let Some(system_prompt) = args.system_prompt.clone().filter(|s| !s.is_empty()) { + vec![OpenAIMessage { + role: "system".to_string(), + content: Some(OpenAIContent::Text(system_prompt)), + ..Default::default() + }] + } else { + vec![] + }; + + // Fetch flow context for input transforms context, chat and memory + let mut flow_context = get_flow_context(db, job).await; + + // Load previous messages from memory for text output mode (only if context length is set) + if matches!(output_type, OutputType::Text) { + if let Some(context_length) = args.messages_context_length.filter(|&n| n > 0) { + if let Some(step_id) = job.flow_step_id.as_deref() { + if let Some(memory_id) = flow_context + .flow_status + .as_ref() + .and_then(|fs| fs.memory_id) + { + // Read messages from memory + match read_from_memory(&job.workspace_id, memory_id, step_id).await { + Ok(Some(loaded_messages)) => { + // Take the last n messages + let start_idx = loaded_messages.len().saturating_sub(context_length); + let mut messages_to_load = loaded_messages[start_idx..].to_vec(); + let first_non_tool_message_index = + messages_to_load.iter().position(|m| m.role != "tool"); + + // Remove the first messages if their role is "tool" to avoid OpenAI API error + if let Some(index) = first_non_tool_message_index { + messages_to_load = messages_to_load[index..].to_vec(); + } + + messages.extend(messages_to_load); + } + Ok(None) => {} + Err(e) => { + tracing::error!("Failed to read memory for step {}: {}", step_id, e); + } + } + } + } + } + } + + // Extract previous step result only if any tool needs it + let previous_result = { + if any_tool_needs_previous_result(&tools) { + if let Some(ref flow_status) = flow_context.flow_status { + get_previous_job_result(db, &job.workspace_id, flow_status) + .await + .ok() + .flatten() + } else { + None + } + } else { + None + } + }; + + // Build IdContext for results.stepId syntax + let id_context = { + if let Some(ref flow_status) = flow_context.flow_status { + // Get the step ID from the AI agent's flow step + let previous_id = job + .flow_step_id + .clone() + .unwrap_or_else(|| "unknown".to_string()); + + Some(get_transform_context(job, &previous_id, flow_status).await?) + } else { + None + } + }; + + // Create user message with optional images + let mut parts = vec![ContentPart::Text { text: args.user_message.clone() }]; + if let Some(images) = &args.user_images { + for image in images.iter() { + if !image.s3.is_empty() { + parts.push(ContentPart::S3Object { s3_object: image.clone() }); + } + } + } + let user_content = OpenAIContent::Parts(parts); + + messages.push(OpenAIMessage { + role: "user".to_string(), + content: Some(user_content), + ..Default::default() + }); + + let mut actions = vec![]; + let mut content = None; + + // Check if this provider supports tools with the current output type + let supports_tools = query_builder.supports_tools_with_output_type(output_type); + + let mut tool_defs: Option> = if tools.is_empty() || !supports_tools { + None + } else { + Some(tools.iter().map(|t| t.def.clone()).collect()) + }; + + // Handle structured output schema + let has_output_properties = args + .output_schema + .as_ref() + .and_then(|schema| schema.properties.as_ref()) + .map(|props| !props.is_empty()) + .unwrap_or(false); + + let is_claude_model = is_claude_model(&args.provider.model); + let mut used_structured_output_tool = false; + let mut structured_output_tool_name: Option = None; + + // For text output with schema, handle structured output + if has_output_properties && output_type == &OutputType::Text { + let schema = args.output_schema.as_ref().unwrap(); + if is_claude_model { + // Anthropic uses a tool for structured output + let unique_tool_name = find_unique_tool_name("structured_output", tool_defs.as_deref()); + structured_output_tool_name = Some(unique_tool_name.clone()); + + let output_tool = ToolDef { + r#type: "function".to_string(), + function: ToolDefFunction { + name: unique_tool_name, + description: Some( + "This tool MUST be used last to return a structured JSON object as the final output." + .to_string(), + ), + parameters: to_raw_value(&schema), + }, + }; + if let Some(ref mut existing_tools) = tool_defs { + existing_tools.push(output_tool); + } else { + tool_defs = Some(vec![output_tool]); + } + } + // For non-Anthropic providers, response_format is handled by the query builder + } + + // Check if streaming is enabled and supported + let should_stream = args.streaming.unwrap_or(false) + && query_builder.supports_streaming() + && output_type == &OutputType::Text; + + *has_stream = should_stream; + + let mut final_events_str = String::new(); + + let stream_event_processor = if should_stream { + Some(StreamEventProcessor::new(conn, job)) + } else { + None + }; + + // Main agent loop + for i in 0..MAX_AGENT_ITERATIONS { + if used_structured_output_tool { + break; + } + + // For text output or image output with tools + let build_args = BuildRequestArgs { + messages: &messages, + tools: tool_defs.as_deref(), + model: args.provider.get_model(), + temperature: args.temperature, + max_tokens: args.max_completion_tokens, + output_schema: args.output_schema.as_ref(), + output_type, + system_prompt: args.system_prompt.as_deref(), + user_message: &args.user_message, + images: args.user_images.as_deref(), + }; + + let request_body = query_builder + .build_request(&build_args, client, &job.workspace_id, should_stream) + .await?; + + let endpoint = + query_builder.get_endpoint(&base_url, args.provider.get_model(), output_type); + let auth_headers = query_builder.get_auth_headers(api_key, &base_url, output_type); + + let timeout = resolve_job_timeout(conn, &job.workspace_id, job.id, job.timeout) + .await + .0; + + let mut request = HTTP_CLIENT + .post(&endpoint) + .timeout(timeout) + .header("Content-Type", "application/json"); + + // Apply authentication headers + for (header_name, header_value) in &auth_headers { + request = request.header(*header_name, header_value.clone()); + } + + // Apply custom headers from AI_HTTP_HEADERS environment variable + for (header_name, header_value) in AI_HTTP_HEADERS.iter() { + request = request.header(header_name.as_str(), header_value.as_str()); + } + + if args.provider.kind.is_azure_openai(&base_url) { + request = request.query(&[("api-version", AZURE_API_VERSION)]) + } + + let resp = request + .body(request_body) + .send() + .await + .map_err(|e| Error::internal_err(format!("Failed to call API: {}", e)))?; + + match resp.error_for_status_ref() { + Ok(_) => { + let parsed = if let Some(stream_event_processor) = stream_event_processor.clone() { + query_builder + .parse_streaming_response(resp, stream_event_processor) + .await? + } else { + // Handle non-streaming response + query_builder.parse_response(resp).await? + }; + + match parsed { + ParsedResponse::Text { content: response_content, tool_calls, events_str } => { + if let Some(events_str) = events_str { + final_events_str.push_str(&events_str); + } + + if let Some(ref response_content) = response_content { + actions.push(AgentAction::Message {}); + messages.push(OpenAIMessage { + role: "assistant".to_string(), + content: Some(OpenAIContent::Text(response_content.clone())), + agent_action: Some(AgentAction::Message {}), + ..Default::default() + }); + + update_flow_status_module_with_actions(db, parent_job, &actions) + .await?; + update_flow_status_module_with_actions_success(db, parent_job, true) + .await?; + + content = Some(OpenAIContent::Text(response_content.clone())); + + // Add assistant message to conversation if chat_input_enabled + let chat_enabled = flow_context + .flow_status + .as_ref() + .and_then(|fs| fs.chat_input_enabled) + .unwrap_or(false); + if chat_enabled && !response_content.is_empty() { + if let Some(memory_id) = flow_context + .flow_status + .as_ref() + .and_then(|fs| fs.memory_id) + { + let agent_job_id = job.id; + let db_clone = db.clone(); + let message_content = response_content.clone(); + let step_name = get_step_name_from_flow( + summary.as_deref(), + job.flow_step_id.as_deref(), + ); + + // Spawn task because we do not need to wait for the result + tokio::spawn(async move { + if let Err(e) = add_message_to_conversation( + &db_clone, + &memory_id, + Some(agent_job_id), + &message_content, + MessageType::Assistant, + &step_name, + true, + ) + .await + { + tracing::warn!("Failed to add assistant message to conversation {}: {}", memory_id, e); + } + }); + } + } + } + + if tool_calls.is_empty() { + break; + } else if i == MAX_AGENT_ITERATIONS - 1 { + return Err(Error::internal_err( + "AI agent reached max iterations, but there are still tool calls" + .to_string(), + )); + } + + messages.push(OpenAIMessage { + role: "assistant".to_string(), + tool_calls: Some(tool_calls.clone()), + ..Default::default() + }); + + // Handle tool calls using extracted tools module + let tool_execution_ctx = ToolExecutionContext { + db, + conn, + job, + parent_job, + summary: &summary, + client, + worker_dir, + base_internal_url, + worker_name, + hostname, + occupancy_metrics, + job_completed_tx, + killpill_rx, + stream_event_processor: stream_event_processor.as_ref(), + flow_context: &mut flow_context, + previous_result: &previous_result, + id_context: &id_context, + }; + + let (tool_messages, tool_content, tool_used_structured_output) = + execute_tool_calls( + tool_execution_ctx, + &tool_calls, + &tools, + mcp_clients, + &mut actions, + &mut final_events_str, + &structured_output_tool_name, + ) + .await?; + + messages.extend(tool_messages); + if let Some(tc) = tool_content { + content = Some(tc); + } + used_structured_output_tool = tool_used_structured_output; + } + ParsedResponse::Image { base64_data } => { + // For image output, upload to S3 and track in conversation + let s3_object = upload_image_to_s3(&base64_data, job, client).await?; + + let content = to_raw_value(&s3_object); + + // Add assistant message to conversation if chat_input_enabled + let chat_enabled = flow_context + .flow_status + .as_ref() + .and_then(|fs| fs.chat_input_enabled) + .unwrap_or(false); + if chat_enabled { + if let Some(memory_id) = flow_context + .flow_status + .as_ref() + .and_then(|fs| fs.memory_id) + { + let agent_job_id = job.id; + let db_clone = db.clone(); + let flow_step_id_owned = job.flow_step_id.clone(); + let summary_owned = summary.map(|s| s.to_string()); + + // Create extended version with type discriminator for conversation storage + // This avoids conflicts with outputs that are of the same format as S3 objects + let s3_with_type = S3ObjectWithType { + s3_object: s3_object.clone(), + r#type: "windmill_s3_object".to_string(), + }; + + let message_content = serde_json::to_string(&s3_with_type) + .unwrap_or_else(|_| content.get().to_string()); + + // Spawn task because we do not need to wait for the result + tokio::spawn(async move { + let step_name = get_step_name_from_flow( + summary_owned.as_deref(), + flow_step_id_owned.as_deref(), + ); + + if let Err(e) = add_message_to_conversation( + &db_clone, + &memory_id, + Some(agent_job_id), + &message_content, + MessageType::Assistant, + &step_name, + true, + ) + .await + { + tracing::warn!("Failed to add assistant message to conversation {}: {}", memory_id, e); + } + }); + } + } + + // Return early since image generation is complete + return Ok(content); + } + } + } + Err(e) => { + let _status = resp.status(); + let text = resp + .text() + .await + .unwrap_or_else(|_| "".to_string()); + return Err(Error::internal_err(format!("API error: {} - {}", e, text))); + } + } + } + + // Return the final result + let final_messages: Vec = messages + .iter() + .map(|m| Message { message: m, agent_action: m.agent_action.as_ref() }) + .collect(); + + // Parse content as JSON for structured output, fallback to string if it fails + let output_value = match content { + Some(content_str) => match has_output_properties { + true => match content_str { + OpenAIContent::Text(text) => { + serde_json::from_str::>(&text).map_err(|_e| { + Error::internal_err(format!("Failed to parse structured output: {}", text)) + }) + } + OpenAIContent::Parts(_parts) => Err(Error::internal_err( + "Failed to parse structured output".to_string(), + )), + }, + false => Ok(match content_str { + OpenAIContent::Text(text) => to_raw_value(&text), + OpenAIContent::Parts(parts) => to_raw_value(&parts), + }), + }?, + None => to_raw_value(&""), + }; + + if let Some(stream_event_processor) = stream_event_processor { + if let Some(handle) = stream_event_processor.to_handle() { + if let Err(e) = handle.await { + return Err(Error::internal_err(format!( + "Error waiting for stream event processor: {}", + e + ))); + } + } + } + + // Persist complete conversation to memory at the end (only if context length is set) + // final_messages contains the complete history (old messages + new ones) + if matches!(output_type, OutputType::Text) { + if let Some(context_length) = args.messages_context_length.filter(|&n| n > 0) { + if let Some(step_id) = job.flow_step_id.as_deref() { + // Extract OpenAIMessages from final_messages + let all_messages: Vec = + final_messages.iter().map(|m| m.message.clone()).collect(); + + if !all_messages.is_empty() { + // Keep only the last n messages + let start_idx = all_messages.len().saturating_sub(context_length); + let messages_to_persist = all_messages[start_idx..].to_vec(); + + if let Some(memory_id) = flow_context.flow_status.and_then(|fs| fs.memory_id) { + if let Err(e) = write_to_memory( + &job.workspace_id, + memory_id, + step_id, + &messages_to_persist, + ) + .await + { + tracing::error!( + "Failed to persist {} messages to memory for step {}: {}", + messages_to_persist.len(), + step_id, + e + ); + } + } + } + } + } + } + + Ok(to_raw_value(&AIAgentResult { + output: output_value, + messages: final_messages, + wm_stream: if !final_events_str.is_empty() { + Some(final_events_str) + } else { + None + }, + })) +} diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 72640eb9da..ac1ae4e233 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -13,6 +13,7 @@ use tokio::process::Command; use uuid::Uuid; use windmill_common::{ error, + git_sync_oss::{prepend_token_to_github_url}, worker::{ is_allowed_file_location, to_raw_value, write_file, write_file_at_user_defined_location, Connection, WORKER_CONFIG, @@ -20,7 +21,9 @@ use windmill_common::{ }; use windmill_queue::MiniPulledJob; -use windmill_parser_yaml::{AnsibleRequirements, GitRepo, ResourceOrVariablePath}; +use windmill_parser_yaml::{ + AnsibleRequirements, GitRepo, PreexistingAnsibleInventory, ResourceOrVariablePath, +}; use windmill_queue::{append_logs, CanceledBy}; use crate::{ @@ -93,6 +96,7 @@ async fn clone_repo( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -112,7 +116,8 @@ async fn clone_repo( .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let checkout_cmd_child = start_child_process(checkout_cmd, GIT_PATH.as_str(), false).await?; + let checkout_cmd_child = + start_child_process(checkout_cmd, GIT_PATH.as_str(), false).await?; handle_child( job_id, conn, @@ -127,6 +132,7 @@ async fn clone_repo( false, &mut Some(occupancy_metrics), None, + None, ) .await?; } @@ -232,6 +238,7 @@ async fn clone_repo_without_history( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -249,7 +256,8 @@ async fn clone_repo_without_history( .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let add_remote_cmd_child = start_child_process(add_remote_cmd, GIT_PATH.as_str(), false).await?; + let add_remote_cmd_child = + start_child_process(add_remote_cmd, GIT_PATH.as_str(), false).await?; handle_child( job_id, conn, @@ -264,6 +272,7 @@ async fn clone_repo_without_history( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -296,6 +305,7 @@ async fn clone_repo_without_history( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -328,6 +338,7 @@ async fn clone_repo_without_history( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -462,6 +473,7 @@ pub async fn install_galaxy_collections( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -484,7 +496,8 @@ pub async fn install_galaxy_collections( .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let child = start_child_process(galaxy_collections_cmd, ANSIBLE_GALAXY_PATH.as_str(), false).await?; + let child = + start_child_process(galaxy_collections_cmd, ANSIBLE_GALAXY_PATH.as_str(), false).await?; handle_child( job_id, conn, @@ -499,6 +512,7 @@ pub async fn install_galaxy_collections( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -874,16 +888,41 @@ pub async fn handle_ansible_job( let inventories: Vec = reqs .as_ref() - .map(|x| { - x.inventories + .map(|x| -> Result, _> { + let mut ret: Vec = x + .inventories .clone() .iter() .flat_map(|i| vec!["-i".to_string(), i.name.clone()].into_iter()) - .collect() + .collect(); + + let additional: Vec = x + .additional_inventories + .iter() + .map(|i| match i { + PreexistingAnsibleInventory::Static(name) => Ok(Some(vec![name.clone()])), + PreexistingAnsibleInventory::PassedInArgs(inv_def) => interpolated_args + .as_ref() + .and_then(|args| args.get(&inv_def.name)) + .and_then(|v| serde_json::from_str(v.get()).transpose()) + .transpose(), + }) + .collect::, _>>()? + .into_iter() + .flatten() + .flatten() + .flat_map(|name| vec!["-i".to_string(), name]) + .collect(); + + ret.extend(additional); + Ok::<_, windmill_common::error::Error>(ret) }) + .transpose()? .unwrap_or_else(|| vec![]); let mut nsjail_extra_mounts = vec![]; + let mut playbook_override = None; + if let Some(r) = reqs.as_ref() { nsjail_extra_mounts = create_file_resources( &job.id, @@ -896,6 +935,106 @@ pub async fn handle_ansible_job( ) .await?; + if let Some(delegated_git_repo) = r.delegate_to_git_repo.as_ref() { + let serde_json::Value::Object(git_repo_resource) = client + .get_resource_value_interpolated::( + &delegated_git_repo.resource, + Some(job.id.to_string()), + ) + .await? + else { + return Err(windmill_common::error::Error::BadRequest( + "Git repository resource is not an object".to_string(), + )); + }; + + let mut secret_url = git_repo_resource.get("url").and_then(|s| s.as_str()).map(|s| s.to_string()) + .ok_or(anyhow!("Failed to get url from git repo resource, please check that the resource has the correct type (git_repository)"))?; + + #[cfg(feature = "enterprise")] + let is_github_app = git_repo_resource.get("is_github_app").and_then(|s| s.as_bool()) + .ok_or(anyhow!("Failed to get `is_github_app` field from git repo resource, please check that the resource has the correct type (git_repository)"))?; + + #[cfg(feature = "enterprise")] + if is_github_app { + if let Connection::Sql(db) = conn { + let token = windmill_common::git_sync_oss::get_github_app_token_internal(db, &client.token).await?; + secret_url = prepend_token_to_github_url(&secret_url, &token)?; + } else { + return Err(windmill_common::error::Error::BadRequest("Github App authentication is currently unavailable for agent workers. Contact the windmill team to request this feature".to_string())); + } + } + + let branch = Some(git_repo_resource.get("branch").and_then(|s| s.as_str()).map(|s| s.to_string()) + .ok_or(anyhow!("Failed to get branch from git repo resource, please check that the resource has the correct type (git_repository)"))?).filter(|s| !s.is_empty()); + + let target_path = "delegate_git_repository".to_string(); + + let repo = + GitRepo { url: secret_url, commit: delegated_git_repo.commit.clone(), branch, target_path }; + append_logs( + &job.id, + &job.workspace_id, + format!("\nCloning {}...\n", delegated_git_repo.resource), + conn, + ) + .await; + if let Some(commit) = delegated_git_repo.commit.as_ref() { + clone_repo_without_history( + &repo, + commit, + job_dir, + &job.id, + worker_name, + conn, + mem_peak, + canceled_by, + &job.workspace_id, + occupancy_metrics, + git_ssh_cmd, + ) + .await + .map_err(|e| anyhow!("Failed to clone git repo `{}`: {e}", repo.url))?; + } else { + clone_repo( + &repo, + job_dir, + &job.id, + worker_name, + conn, + mem_peak, + canceled_by, + &job.workspace_id, + occupancy_metrics, + git_ssh_cmd, + ) + .await + .map_err(|e| anyhow!("Failed to clone git repo `{}`: {e}", repo.url))?; + } + + append_logs( + &job.id, + &job.workspace_id, + format!( + "Cloned {} into {}\n", + delegated_git_repo.resource, &repo.target_path + ), + conn, + ) + .await; + + playbook_override = Some( + delegated_git_repo + .playbook + .as_ref() + .map(|p| format!("{}/{}", &repo.target_path, p)), + ); + } + + if playbook_override.clone().flatten().is_none() && playbook.is_empty() { + return Err(windmill_common::error::Error::BadRequest("No playbook was specified. Append a playbook to your script or specify one in the delegate_to_git_repo -> playbook section.".to_string())); + } + for repo in &r.git_repos { append_logs( &job.id, @@ -1049,7 +1188,10 @@ mount {{ reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders); } - let mut cmd_args = vec!["main.yml", "--extra-vars", "@args.json"]; + let playbook = playbook_override + .flatten() + .unwrap_or("main.yml".to_string()); + let mut cmd_args = vec![playbook.as_str(), "--extra-vars", "@args.json"]; cmd_args.extend(inventories.iter().map(|s| s.as_str())); cmd_args.extend(cmd_options.iter().map(|s| s.as_str())); @@ -1133,6 +1275,7 @@ fi false, &mut Some(occupancy_metrics), None, + None, ) .await?; read_and_check_result(job_dir).await diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index b224957958..d147aaff9b 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, fs, process::Stdio}; +use std::{collections::HashMap, process::Stdio}; #[cfg(feature = "dind")] use bollard::container::{ @@ -29,12 +29,6 @@ lazy_static::lazy_static! { pub static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| "/bin/bash".to_string()); } const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto"); -const NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT: &str = - include_str!("../nsjail/run.powershell.config.proto"); - -lazy_static::lazy_static! { - static ref RE_POWERSHELL_IMPORTS: Regex = Regex::new(r#"^Import-Module\s+(?:-Name\s+)?"?([^-\s"]+)"?"#).unwrap(); -} #[cfg(feature = "dind")] use crate::handle_child::run_future_with_polling_update_job_poller; @@ -45,19 +39,23 @@ use crate::{ OccupancyMetrics, }, handle_child::handle_child, - DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, POWERSHELL_CACHE_DIR, - POWERSHELL_PATH, PROXY_ENVS, TZ_ENV, + DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, }; use windmill_common::client::AuthedClient; -#[cfg(windows)] -use crate::SYSTEM_ROOT; - lazy_static::lazy_static! { pub static ref ANSI_ESCAPE_RE: Regex = Regex::new(r"\x1b\[[0-9;]*m").unwrap(); } +fn raw_to_string(x: &str) -> String { + match serde_json::from_str::(x) { + Ok(serde_json::Value::String(x)) => x, + Ok(x) => serde_json::to_string(&x).unwrap_or_else(|_| String::new()), + _ => String::new(), + } +} + #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_bash_job( mem_peak: &mut i32, @@ -231,6 +229,7 @@ exit $exit_status true, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -510,395 +509,3 @@ async fn container_is_alive(client: &bollard::Docker, container_id: &str) -> boo false } } - -fn raw_to_string(x: &str) -> String { - match serde_json::from_str::(x) { - Ok(serde_json::Value::String(x)) => x, - Ok(x) => serde_json::to_string(&x).unwrap_or_else(|_| String::new()), - _ => String::new(), - } -} - -const POWERSHELL_INSTALL_CODE: &str = r#" -$availableModules = Get-Module -ListAvailable -$path = '{path}' - -$moduleNames = @({modules}) - -foreach ($module in $moduleNames) { - if (-not ($availableModules | Where-Object { $_.Name -eq $module })) { - Write-Host "Installing module $module..." - Save-Module -Name $module -Path $path -Force - } else { - Write-Host "Module $module already installed" - } -} -"#; - -#[tracing::instrument(level = "trace", skip_all)] -pub async fn handle_powershell_job( - mem_peak: &mut i32, - canceled_by: &mut Option, - job: &MiniPulledJob, - db: &Connection, - client: &AuthedClient, - parent_runnable_path: Option, - content: &str, - job_dir: &str, - shared_mount: &str, - base_internal_url: &str, - worker_name: &str, - envs: HashMap, - occupancy_metrics: &mut OccupancyMetrics, -) -> Result, Error> { - let pwsh_args = { - let args = build_args_map(job, client, &db).await?.map(Json); - let job_args = if args.is_some() { - args.as_ref() - } else { - job.args.as_ref() - }; - - let args_owned = windmill_parser_bash::parse_powershell_sig(&content)? - .args - .iter() - .map(|arg| { - ( - arg.name.clone(), - job_args - .and_then(|x| x.get(&arg.name).map(|x| raw_to_string(x.get()))) - .unwrap_or_else(String::new), - ) - }) - .collect::>(); - args_owned - .iter() - .map(|(n, v)| vec![format!("--{n}"), format!("{v}")]) - .flatten() - .collect::>() - }; - - #[cfg(windows)] - let split_char = '\\'; - - #[cfg(unix)] - let split_char = '/'; - - let installed_modules = fs::read_dir(POWERSHELL_CACHE_DIR)? - .filter_map(|x| { - x.ok().map(|x| { - x.path() - .display() - .to_string() - .split(split_char) - .last() - .unwrap_or_default() - .to_lowercase() - }) - }) - .collect::>(); - - let mut modules_to_install: Vec = Vec::new(); - let mut logs1 = String::new(); - for line in content.lines() { - for cap in RE_POWERSHELL_IMPORTS.captures_iter(line) { - let module = cap.get(1).unwrap().as_str(); - if !installed_modules.contains(&module.to_lowercase()) { - modules_to_install.push(module.to_string()); - } else { - logs1.push_str(&format!("\n{} found in cache", module.to_string())); - } - } - } - - if !logs1.is_empty() { - append_logs(&job.id, &job.workspace_id, logs1, db).await; - } - - if !modules_to_install.is_empty() { - let install_string = POWERSHELL_INSTALL_CODE - .replace("{path}", POWERSHELL_CACHE_DIR) - .replace( - "{modules}", - &modules_to_install - .iter() - .map(|x| format!("'{x}'")) - .collect::>() - .join(", "), - ); - let mut cmd = Command::new(POWERSHELL_PATH.as_str()); - cmd.args(&["-Command", &install_string]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let child = start_child_process(cmd, POWERSHELL_PATH.as_str(), false).await?; - - handle_child( - &job.id, - db, - mem_peak, - canceled_by, - child, - false, - worker_name, - &job.workspace_id, - "powershell install", - job.timeout, - false, - &mut Some(occupancy_metrics), - None, - ) - .await?; - } - - let mut logs2 = "".to_string(); - logs2.push_str("\n\n--- POWERSHELL CODE EXECUTION ---\n"); - append_logs(&job.id, &job.workspace_id, logs2, db).await; - - // make sure default (only allhostsallusers) modules are loaded, disable autoload (cache can be large to explore especially on cloud) and add /tmp/windmill/cache to PSModulePath - #[cfg(unix)] - let profile = format!( - "$PSModuleAutoloadingPreference = 'None' -$PSModulePathBackup = $env:PSModulePath -$env:PSModulePath = \"$PSHome/Modules\" -Get-Module -ListAvailable | Import-Module -$env:PSModulePath = \"{}:$PSModulePathBackup\"", - POWERSHELL_CACHE_DIR - ); - - #[cfg(windows)] - let profile = format!( - "$PSModuleAutoloadingPreference = 'None' -$PSModulePathBackup = $env:PSModulePath -$env:PSModulePath = \"C:\\Program Files\\PowerShell\\7\\Modules\" -Get-Module -ListAvailable | Import-Module -$env:PSModulePath = \"{};$PSModulePathBackup\"", - POWERSHELL_CACHE_DIR - ); - - // NOTE: powershell error handling / termination is quite tricky compared to bash - // here we're trying to catch terminating errors and propagate the exit code - // to the caller such that the job will be marked as failed. It's up to the user - // to catch specific errors in their script not caught by the below as there is no - // generic set -eu as in bash - let strict_termination_start = "$ErrorActionPreference = 'Stop'\n\ - Set-StrictMode -Version Latest\n\ - try {\n"; - - let strict_termination_end = "\n\ - } catch {\n\ - Write-Output \"An error occurred:\n\"\ - Write-Output $_ - exit 1\n\ - }\n"; - - // make sure param() is first - let param_match = windmill_parser_bash::RE_POWERSHELL_PARAM.find(&content); - let content: String = if let Some(param_match) = param_match { - let param_match = param_match.as_str(); - format!( - "{}\n{}\n{}\n{}\n{}", - param_match, - profile, - strict_termination_start, - content.replace(param_match, ""), - strict_termination_end - ) - } else { - format!("{}\n{}", profile, content) - }; - - write_file(job_dir, "main.ps1", content.as_str())?; - - #[cfg(unix)] - write_file( - job_dir, - "wrapper.sh", - &format!("set -o pipefail\nset -e\nmkfifo bp\ncat bp | tail -1 > ./result2.out &\n{} -F ./main.ps1 \"$@\" 2>&1 | tee bp\nwait $!", POWERSHELL_PATH.as_str()), - )?; - - #[cfg(windows)] - write_file( - job_dir, - "wrapper.ps1", - &format!( - "param([string[]]$args)\n\ - $ErrorActionPreference = 'Stop'\n\ - $pipe = New-TemporaryFile\n\ - & \"{}\" -File ./main.ps1 @args 2>&1 | Tee-Object -FilePath $pipe\n\ - Get-Content -Path $pipe | Select-Object -Last 1 | Set-Content -Path './result2.out'\n\ - Remove-Item $pipe\n\ - exit $LASTEXITCODE\n", - POWERSHELL_PATH.as_str() - ), - )?; - - let mut reserved_variables = - get_reserved_variables(job, &client.token, db, parent_runnable_path).await?; - reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); - - let _ = write_file(job_dir, "result.json", "")?; - let _ = write_file(job_dir, "result.out", "")?; - let _ = write_file(job_dir, "result2.out", "")?; - - let nsjail = !*DISABLE_NSJAIL - && job - .runnable_path - .as_ref() - .map(|x| { - !x.starts_with(INIT_SCRIPT_PATH_PREFIX) - && !x.starts_with(PERIODIC_SCRIPT_PATH_PREFIX) - }) - .unwrap_or(true); - let child = if nsjail { - let _ = write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{SHARED_MOUNT}", shared_mount) - .replace("{CACHE_DIR}", POWERSHELL_CACHE_DIR), - )?; - let mut cmd_args = vec![ - "--config", - "run.config.proto", - "--", - BIN_BASH.as_str(), - "wrapper.sh", - ]; - cmd_args.extend(pwsh_args.iter().map(|x| x.as_str())); - let mut cmd = Command::new(NSJAIL_PATH.as_str()); - cmd.current_dir(job_dir) - .env_clear() - .envs(PROXY_ENVS.clone()) - .envs(reserved_variables) - .env("TZ", TZ_ENV.as_str()) - .env("PATH", PATH_ENV.as_str()) - .env("BASE_INTERNAL_URL", base_internal_url) - .args(cmd_args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - start_child_process(cmd, NSJAIL_PATH.as_str(), false).await? - } else { - let mut cmd; - let mut cmd_args; - - #[cfg(unix)] - { - cmd_args = vec!["wrapper.sh"]; - cmd_args.extend(pwsh_args.iter().map(|x| x.as_str())); - cmd = Command::new(BIN_BASH.as_str()); - } - - #[cfg(windows)] - { - cmd_args = vec![r".\wrapper.ps1".to_string()]; - cmd_args.extend(pwsh_args.iter().map(|x| x.replace("--", "-"))); - cmd = Command::new(POWERSHELL_PATH.as_str()); - } - - cmd.current_dir(job_dir) - .env_clear() - .envs(envs) - .envs(reserved_variables) - .env("TZ", TZ_ENV.as_str()) - .env("PATH", PATH_ENV.as_str()) - .env("BASE_INTERNAL_URL", base_internal_url) - .env("HOME", HOME_ENV.as_str()) - .args(&cmd_args) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - #[cfg(windows)] - { - cmd.env("SystemRoot", SYSTEM_ROOT.as_str()) - .env("WINDIR", SYSTEM_ROOT.as_str()) - .env( - "LOCALAPPDATA", - std::env::var("LOCALAPPDATA") - .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())), - ) - .env( - "ProgramData", - std::env::var("ProgramData") - .unwrap_or_else(|_| String::from("C:\\ProgramData")), - ) - .env( - "ProgramFiles", - std::env::var("ProgramFiles") - .unwrap_or_else(|_| String::from("C:\\Program Files")), - ) - .env( - "ProgramFiles(x86)", - std::env::var("ProgramFiles(x86)") - .unwrap_or_else(|_| String::from("C:\\Program Files (x86)")), - ) - .env( - "ProgramW6432", - std::env::var("ProgramW6432") - .unwrap_or_else(|_| String::from("C:\\Program Files")), - ) - .env( - "TMP", - std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), - ) - .env( - "PATHEXT", - std::env::var("PATHEXT").unwrap_or_else(|_| { - String::from(".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL") - }), - ) - .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); - } - - start_child_process(cmd, POWERSHELL_PATH.as_str(), false).await? - }; - - handle_child( - &job.id, - db, - mem_peak, - canceled_by, - child, - !*DISABLE_NSJAIL, - worker_name, - &job.workspace_id, - "powershell run", - job.timeout, - false, - &mut Some(occupancy_metrics), - None, - ) - .await?; - - let result_json_path = format!("{job_dir}/result.json"); - if let Ok(metadata) = tokio::fs::metadata(&result_json_path).await { - if metadata.len() > 0 { - return Ok(read_file(&result_json_path).await?); - } - } - - let result_out_path = format!("{job_dir}/result.out"); - if let Ok(metadata) = tokio::fs::metadata(&result_out_path).await { - if metadata.len() > 0 { - let result = read_file_content(&result_out_path).await?; - return Ok(to_raw_value(&json!(result))); - } - } - - let result_out_path2 = format!("{job_dir}/result2.out"); - if tokio::fs::metadata(&result_out_path2).await.is_ok() { - let result = read_file_content(&result_out_path2) - .await? - .trim() - .to_string(); - return Ok(to_raw_value(&json!(result))); - } - - Ok(to_raw_value(&json!( - "No result.out, result2.out or result.json found" - ))) -} diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index ed6814f34f..cf93736850 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -19,7 +19,8 @@ use serde::Deserialize; use crate::common::{build_args_values, resolve_job_timeout}; use crate::common::{ - build_http_client, s3_mode_args_to_worker_data, OccupancyMetrics, S3ModeWorkerData, + build_http_client, get_reserved_variables, s3_mode_args_to_worker_data, OccupancyMetrics, + S3ModeWorkerData, }; use crate::handle_child::run_future_with_polling_update_job_poller; use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; @@ -313,6 +314,7 @@ pub async fn do_bigquery( worker_name: &str, column_order: &mut Option>, occupancy_metrics: &mut OccupancyMetrics, + parent_runnable_path: Option, ) -> windmill_common::error::Result> { let bigquery_args = build_args_values(job, client, conn).await?; @@ -364,8 +366,15 @@ pub async fn do_bigquery( .map_err(|x| Error::ExecutionErr(x.to_string()))? .args; - let (query, args_to_skip) = - &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &bigquery_args)?; + let reserved_variables = + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; + + let (query, args_to_skip) = &sanitize_and_interpolate_unsafe_sql_args( + query, + &sig, + &bigquery_args, + &reserved_variables, + )?; let queries = parse_sql_blocks(query); diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index dce084bdf4..757656b934 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -18,13 +18,18 @@ use crate::{ common::{ create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_file_content, read_result, start_child_process, write_file_binary, OccupancyMetrics, + StreamNotifier, }, handle_child::handle_child, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_NO_CACHE, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TZ_ENV, }; -use windmill_common::client::AuthedClient; +use windmill_common::{ + client::AuthedClient, + s3_helpers::BundleFormat, + scripts::{id_to_codebase_info, CodebaseInfo}, +}; #[cfg(windows)] use crate::SYSTEM_ROOT; @@ -168,6 +173,7 @@ pub async fn gen_bun_lockfile( false, occupancy_metrics, None, + None, ) .await?; } else { @@ -307,6 +313,7 @@ pub async fn install_bun_lockfile( .env_clear() .envs(PROXY_ENVS.clone()) .envs(common_bun_proc_envs) + .envs(&*crate::worker::WHITELIST_ENVS) .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -374,6 +381,7 @@ pub async fn install_bun_lockfile( false, occupancy_metrics, None, + None, ) .await?; } else { @@ -546,6 +554,7 @@ pub async fn generate_wrapper_mjs( false, occupancy_metrics, None, + None, ) .await?; fs::rename( @@ -597,6 +606,7 @@ pub async fn generate_bun_bundle( false, occupancy_metrics, None, + None, ) .await?; } else { @@ -605,15 +615,18 @@ pub async fn generate_bun_bundle( Ok(()) } -pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { +struct PulledCodebase { + is_esm: bool, +} +async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result { let path = windmill_common::s3_helpers::bundle(&w_id, &id); let bun_cache_path = format!( "{}/{}", windmill_common::worker::ROOT_CACHE_NOMOUNT_DIR, path ); - let is_tar = id.ends_with(".tar"); + let CodebaseInfo { is_tar, is_esm } = id_to_codebase_info(id); let dst = format!( "{job_dir}/{}", if is_tar { "codebase.tar" } else { "main.js" } @@ -634,9 +647,9 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { && object_store.is_none() { let bun_cache_path = format!( - "{}{}", + "{}/{}", *windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR, - id + path ); if std::fs::metadata(&bun_cache_path).is_ok() { tracing::info!("loading {bun_cache_path} from standalone bundle cache"); @@ -666,7 +679,7 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { } } - Ok(()) + Ok(PulledCodebase { is_esm }) } fn extract_saved_codebase( @@ -853,6 +866,7 @@ pub async fn handle_bun_job( new_args: &mut Option>>, occupancy_metrics: &mut OccupancyMetrics, precomputed_agent_info: Option, + has_stream: &mut bool, ) -> error::Result> { let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); @@ -901,13 +915,11 @@ pub async fn handle_bun_job( let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(Some(&base_internal_url)).await; - if codebase.is_some() { - annotation.nodejs = true - } let main_override = job.script_entrypoint_override.as_deref(); let apply_preprocessor = job.flow_step_id.as_deref() != Some("preprocessor") && job.preprocessed == Some(false); + let mut format = BundleFormat::Cjs; if has_bundle_cache { let target; let symlink; @@ -929,7 +941,10 @@ pub async fn handle_bun_job( )) })?; } else if let Some(codebase) = codebase.as_ref() { - pull_codebase(&job.workspace_id, codebase, job_dir).await?; + let pulled_codebase = pull_codebase(&job.workspace_id, codebase, job_dir).await?; + if pulled_codebase.is_esm { + format = BundleFormat::Esm; + } } else if let Some(reqs) = requirements_o.as_ref() { let (pkg, lock, empty, is_binary) = split_lockfile(reqs); @@ -985,6 +1000,10 @@ pub async fn handle_bun_job( // } } + if codebase.is_some() && format == BundleFormat::Cjs { + annotation.nodejs = true + } + let mut init_logs = if annotation.native { "\n\n--- NATIVE CODE EXECUTION ---\n".to_string() } else if has_bundle_cache { @@ -994,7 +1013,11 @@ pub async fn handle_bun_job( "\n\n--- BUN BUNDLE SNAPSHOT EXECUTION ---\n".to_string() } } else if codebase.is_some() { - "\n\n--- NODE CODEBASE SNAPSHOT EXECUTION ---\n".to_string() + if format == BundleFormat::Esm { + "\n\n--- ESM CODEBASE SNAPSHOT EXECUTION ---\n".to_string() + } else { + "\n\n--- CJS CODEBASE SNAPSHOT EXECUTION ---\n".to_string() + } } else if annotation.native { "\n\n--- NATIVE CODE EXECUTION ---\n".to_string() } else if annotation.nodejs { @@ -1110,7 +1133,7 @@ async function run() {{ let res = await Main.{main_name}(...argsArr); if (isAsyncIterable(res)) {{ for await (const chunk of res) {{ - console.log("WM_STREAM: " + chunk.replace('\n', '\\n')); + console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n')); }} res = null; }} @@ -1308,6 +1331,8 @@ try {{ append_logs(&job.id, &job.workspace_id, format!("{init_logs}\n"), conn).await; + let stream_notifier = StreamNotifier::new(conn, job); + let result = crate::js_eval::eval_fetch_timeout( env_code, inner_content.clone(), @@ -1323,6 +1348,8 @@ try {{ &job.workspace_id, false, occupancy_metrics, + stream_notifier, + has_stream, ) .await?; tracing::info!( @@ -1465,6 +1492,8 @@ try {{ .await? }; + let stream_notifier = StreamNotifier::new(conn, job); + let handle_result = handle_child( &job.id, conn, @@ -1479,9 +1508,12 @@ try {{ false, &mut Some(occupancy_metrics), None, + stream_notifier, ) .await?; + *has_stream = handle_result.result_stream.is_some(); + if apply_preprocessor { let args = read_file(&format!("{job_dir}/args.json")) .await @@ -1593,12 +1625,17 @@ pub async fn start_worker( None, None, None, + None, ) .await; let context_envs = build_envs_map(context.to_vec()).await; + let mut format = BundleFormat::Cjs; if let Some(codebase) = codebase.as_ref() { - pull_codebase(w_id, codebase, job_dir).await?; + let pulled_codebase = pull_codebase(w_id, codebase, job_dir).await?; + if pulled_codebase.is_esm { + format = BundleFormat::Esm; + } } else if let Some(reqs) = requirements_o { let (pkg, lock, empty, is_binary) = split_lockfile(&reqs); if lock.is_none() { @@ -1725,7 +1762,11 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) { write_file(job_dir, "wrapper.mjs", &wrapper_content)?; } - if !codebase.is_some() { + if format == BundleFormat::Esm { + annotation.nodejs = false; + } + + if !codebase.is_some() || format == BundleFormat::Esm { build_loader( job_dir, base_internal_url, diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 723d99e062..892a3c8b7e 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -14,6 +14,7 @@ use sqlx::{Pool, Postgres}; use tokio::process::Command; use tokio::{fs::File, io::AsyncReadExt}; +use windmill_common::flows::Step; #[cfg(feature = "parquet")] use windmill_common::s3_helpers::{ get_etag_or_empty, LargeFileStorage, ObjectStoreResource, S3Object, @@ -27,13 +28,15 @@ use windmill_common::{ cache::{Cache, RawData}, error::{self, Error}, scripts::ScriptHash, + utils::configure_client, variables::ContextualVariable, }; use anyhow::{anyhow, Result}; use windmill_parser_sql::{s3_mode_extension, S3ModeArgs, S3ModeFormat}; -use windmill_queue::MiniPulledJob; +use windmill_queue::{MiniCompletedJob, MiniPulledJob}; +use std::collections::HashSet; use std::path::Path; use std::{collections::HashMap, sync::Arc, time::Duration}; @@ -233,7 +236,8 @@ pub async fn transform_json_value( } Value::String(y) if y.starts_with("$res:") => { let path = y.strip_prefix("$res:").unwrap(); - if path.split("/").count() < 2 { + + if path.split("/").count() < 2 && !path.starts_with("INSTANCE_DUCKLAKE_CATALOG/") { return Err(Error::internal_err(format!( "Argument `{name}` is an invalid resource path: {path}", ))); @@ -261,7 +265,11 @@ pub async fn transform_json_value( ) .await?; decrypt(&mc, encrypted.to_string()).and_then(|x| { - serde_json::from_str(&x).map_err(|e| Error::internal_err(e.to_string())) + serde_json::from_str(&x).map_err(|e| { + Error::internal_err(format!( + "Failed to decrypt '$encrypted:' value: {e}" + )) + }) }) } Connection::Http(_) => { @@ -445,6 +453,7 @@ pub async fn get_reserved_variables( Some(get_root_job_id(job).to_string()), Some(job.scheduled_for.clone()), job.runnable_id, + job.permissioned_as_end_user_email.clone(), ) .await .to_vec(); @@ -660,13 +669,16 @@ pub async fn resolve_job_timeout( ) -> (Duration, Option, bool) { let mut warn_msg: Option = None; #[cfg(feature = "cloud")] - let cloud_premium_workspace = *CLOUD_HOSTED + let cloud_premium_workspace = + *CLOUD_HOSTED && windmill_common::workspaces::get_team_plan_status( _conn.as_sql().expect("cloud cannot use http connection"), _w_id, ) .await - .premium; + .inspect_err(|err| tracing::error!("Failed to get team plan status to resolve job timeout for workspace {_w_id}: {err:#}")) + .map(|s| s.premium) + .unwrap_or(true); #[cfg(not(feature = "cloud"))] let cloud_premium_workspace = false; @@ -961,7 +973,7 @@ pub async fn get_cached_resource_value_if_valid( pub async fn save_in_cache( db: &Pool, _client: &AuthedClient, - job: &MiniPulledJob, + job: &MiniCompletedJob, cached_path: String, r: Arc>, ) { @@ -988,7 +1000,7 @@ pub async fn save_in_cache( "INSERT INTO resource (workspace_id, path, value, resource_type, created_by, edited_at) VALUES ($1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, path) - DO UPDATE SET value = $3, edited_at = now()", + DO UPDATE SET value = EXCLUDED.value, edited_at = now()", job.workspace_id, &cached_path, raw_json as Json<&CachedResource>, @@ -1045,12 +1057,14 @@ pub fn use_flow_root_path(flow_path: &str) -> String { } pub fn build_http_client(timeout_duration: std::time::Duration) -> error::Result { - reqwest::ClientBuilder::new() - .user_agent("windmill/beta") - .timeout(timeout_duration) - .connect_timeout(std::time::Duration::from_secs(10)) - .build() - .map_err(|e| Error::internal_err(format!("Error building http client: {e:#}"))) + configure_client( + reqwest::ClientBuilder::new() + .user_agent("windmill/beta") + .timeout(timeout_duration) + .connect_timeout(std::time::Duration::from_secs(10)), + ) + .build() + .map_err(|e| Error::internal_err(format!("Error building http client: {e:#}"))) } pub fn get_root_job_id(job: &MiniPulledJob) -> uuid::Uuid { @@ -1061,6 +1075,218 @@ pub fn get_root_job_id(job: &MiniPulledJob) -> uuid::Uuid { .unwrap_or(job.id) } +#[derive(Clone)] +pub struct StreamNotifier { + db: DB, + job_id: uuid::Uuid, + parent_job: uuid::Uuid, + root_job: uuid::Uuid, + flow_step_id: Option, +} + +// Helper struct to hold parent job information +#[derive(Debug)] +struct JobInfo { + flow_step_id: Option, + step: Option, + len: Option, + is_branch_one: Option, + next_parent: Option, +} + +// Shared helper function to fetch parent job info with flow status +async fn get_job_info(db: &DB, job_id: Uuid) -> error::Result { + sqlx::query_as!( + JobInfo, + r#"SELECT + flow_step_id, + (flow_status->'step')::integer as step, + jsonb_array_length(flow_status->'modules') as len, + runnable_path ~ '/branchone-\d+$' as is_branch_one, + parent_job as next_parent + FROM v2_job + LEFT JOIN v2_job_status USING (id) + WHERE v2_job.id = $1"#, + job_id + ) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("fetching parent job info: {e:#}"))) +} + +async fn check_if_last_step( + db: &DB, + mut next_parent: Option, + root_job: Uuid, +) -> error::Result { + let mut visited = HashSet::new(); + loop { + // Get parent of current job + let Some(parent_job) = next_parent else { + return Ok(false); + }; + + // Check for cycles + if !visited.insert(parent_job) { + return Ok(false); + } + + let parent_info = get_job_info(db, parent_job).await?; + + if let Some(step) = parent_info.step { + let step = Step::from_i32_and_len(step, parent_info.len.unwrap_or(0) as usize); + if step.is_last_step() { + if parent_job == root_job { + return Ok(true); + } else if parent_info.is_branch_one.unwrap_or(false) { + next_parent = parent_info.next_parent; + continue; + } + } + } + + return Ok(false); + } +} + +// Iterative implementation to avoid stack overflow from async_recursion +// Checks if we're at last step in nested branches AND if any parent's flow_step_id matches early_return_id +async fn check_if_early_return_or_last_in_early_return_parent( + db: &DB, + mut next_parent: Option, + mut step_id: Option, + early_return_id: &str, + root_job: Uuid, +) -> error::Result { + let mut visited = HashSet::new(); + loop { + if step_id.is_none() { + return Ok(false); + } + + let Some(parent_job) = next_parent else { + return Ok(false); + }; + + // Check for cycles + if !visited.insert(parent_job) { + return Ok(false); + } + + // If the parent's flow_step_id matches early_return_id, we found it! + if step_id.as_deref() == Some(early_return_id) && parent_job == root_job { + return Ok(true); + } else { + let parent_info = get_job_info(db, parent_job).await?; + if let Some(step) = parent_info.step { + let step = Step::from_i32_and_len(step, parent_info.len.unwrap_or(0) as usize); + // we only continue if we are at the last step and the parent is a branch one + if step.is_last_step() && parent_info.is_branch_one.unwrap_or(false) { + next_parent = parent_info.next_parent; + step_id = parent_info.flow_step_id; + continue; + } + } + return Ok(false); + } + } +} + +impl StreamNotifier { + pub fn new(conn: &Connection, job: &MiniPulledJob) -> Option { + let root_job = get_root_job_id(job); + if job.is_flow_step() && job.parent_job.is_some() { + match conn { + Connection::Sql(db) => Some(Self { + db: db.clone(), + parent_job: job.parent_job.unwrap(), + job_id: job.id, + root_job, + flow_step_id: job.flow_step_id.clone(), + }), + Connection::Http(_) => { + tracing::warn!( + "Flow job streaming is only supported for workers connected to a database" + ); + None + } + } + } else { + None + } + } + + async fn update_flow_status_with_stream_job_inner( + db: DB, + parent_job: Uuid, + job_id: Uuid, + root_job: Uuid, + flow_step_id: Option, + ) -> Result<(), Error> { + // Check if early_return is set at the flow level + let early_return_node_id = sqlx::query_scalar!( + r#" + SELECT fv.value->>'early_return' as "early_return" + FROM v2_job j + INNER JOIN flow_version fv ON fv.id = j.runnable_id + WHERE j.id = $1 + "#, + root_job + ) + .fetch_optional(&db) + .await? + .flatten(); + + let should_set_stream_job = if let Some(ref early_return_id) = early_return_node_id { + check_if_early_return_or_last_in_early_return_parent( + &db, + Some(parent_job), + flow_step_id, + early_return_id, + root_job, + ) + .await? + } else { + check_if_last_step(&db, Some(parent_job), root_job).await? + }; + + if should_set_stream_job { + sqlx::query!(r#" + UPDATE v2_job_status + SET flow_status = jsonb_set(flow_status, array['stream_job'], to_jsonb($1::UUID::TEXT)) + WHERE id = $2"#, + job_id, + root_job + ) + .execute(&db) + .await?; + } + + Ok(()) + } + + pub fn update_flow_status_with_stream_job(&self) -> () { + let db = self.db.clone(); + let parent_job = self.parent_job; + let job_id = self.job_id; + let root_job = self.root_job; + let flow_step_id = self.flow_step_id.clone(); + tokio::spawn(async move { + if let Err(err) = Self::update_flow_status_with_stream_job_inner( + db, + parent_job, + job_id, + root_job, + flow_step_id, + ) + .await + { + tracing::error!("Could not notify about stream job {}: {err:#?}", parent_job); + } + }); + } +} + #[derive(Clone)] pub struct S3ModeWorkerData { pub client: AuthedClient, @@ -1120,4 +1346,3 @@ pub fn s3_mode_args_to_worker_data( workspace_id: job.workspace_id.clone(), } } - diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index 84cc3184a3..51bdf2f8e8 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -129,6 +129,7 @@ pub async fn generate_nuget_lockfile( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -386,6 +387,7 @@ async fn build_cs_proj( false, &mut Some(occupancy_metrics), None, + None, ) .await?; append_logs(job_id, w_id, "\n\n", conn).await; @@ -643,6 +645,7 @@ pub async fn handle_csharp_job( false, &mut Some(occupancy_metrics), None, + None, ) .await?; read_result(job_dir, None).await diff --git a/backend/windmill-worker/src/dedicated_worker.rs b/backend/windmill-worker/src/dedicated_worker.rs index 1805b22230..e65655c94d 100644 --- a/backend/windmill-worker/src/dedicated_worker.rs +++ b/backend/windmill-worker/src/dedicated_worker.rs @@ -76,7 +76,7 @@ pub async fn handle_dedicated_process( ) -> std::result::Result<(), error::Error> { //do not cache local dependencies - use windmill_queue::{JobCompleted, MiniPulledJob}; + use windmill_queue::{JobCompleted, MiniCompletedJob}; use crate::{handle_child::process_status, PROXY_ENVS}; let cmd_name = format!("dedicated {command_path}"); @@ -133,8 +133,7 @@ pub async fn handle_dedicated_process( } }); - let mut jobs: VecDeque> = - VecDeque::with_capacity(MAX_BUFFERED_DEDICATED_JOBS); + let mut jobs: VecDeque = VecDeque::with_capacity(MAX_BUFFERED_DEDICATED_JOBS); // let mut i = 0; // let mut j = 0; let mut alive = true; @@ -179,21 +178,21 @@ pub async fn handle_dedicated_process( } tracing::debug!("processed job: |{line}|"); if line.starts_with("wm_res[") { - let job: Arc = jobs.pop_front().expect("pop"); + let job = jobs.pop_front().expect("pop"); tracing::info!("job completed on dedicated worker {script_path}: {}", job.id); match serde_json::from_str::>(&line.replace("wm_res[success]:", "").replace("wm_res[error]:", "")) { Ok(result) => { let result = Arc::new(result); append_logs(&job.id, &job.workspace_id, logs.clone(), &db.into()).await; if line.starts_with("wm_res[success]:") { - job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None }, true).await.unwrap() + job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None, has_stream: Some(false), from_cache: None }, true).await.unwrap() } else { - job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None }, true).await.unwrap() + job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None, has_stream: Some(false), from_cache: None }, true).await.unwrap() } }, Err(e) => { tracing::error!("Could not deserialize job result `{line}`: {e:?}"); - job_completed_tx.send_job(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None }, true).await.unwrap(); + job_completed_tx.send_job(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None, has_stream: Some(false), from_cache: None }, true).await.unwrap(); }, }; logs = init_log.clone(); @@ -215,12 +214,15 @@ pub async fn handle_dedicated_process( job = conditional_polling(jobs_rx.recv(), alive && jobs.len() < MAX_BUFFERED_DEDICATED_JOBS) => { // i += 1; if let Some(job) = job { - jobs.push_back(job.clone()); - tracing::info!("received job and adding to queue on dedicated worker for {script_path}: {} (queue_size: {})", job.id, jobs.len()); + let id = job.id; + let args = serde_json::to_string(&job.args).expect("serialize"); + jobs.push_back(MiniCompletedJob::from(job)); + tracing::info!("received job and adding to queue on dedicated worker for {script_path}: {} (queue_size: {})", id, jobs.len()); // write_stdin(&mut stdin, &serde_json::to_string(&job.args.unwrap_or_else(|| serde_json::json!({"x": job.id}))).expect("serialize")).await?; - write_stdin(&mut stdin, &serde_json::to_string(&job.args).expect("serialize")).await?; + write_stdin(&mut stdin, &args).await?; stdin.flush().await.context("stdin flush")?; + // tracing::info!("wrote job to stdin for {script_path}: {} (queue_size: {})", id, jobs.len()); } else { tracing::debug!("job channel closed"); alive = false; diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 431a457d74..cda1f78fdc 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -8,7 +8,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_result, - start_child_process, OccupancyMetrics, + start_child_process, OccupancyMetrics, StreamNotifier, }, handle_child::handle_child, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, @@ -161,6 +161,7 @@ pub async fn generate_deno_lock( false, occupancy_metrics, None, + None, ) .await?; } else { @@ -193,6 +194,7 @@ pub async fn handle_deno_job( envs: HashMap, new_args: &mut Option>>, occupancy_metrics: &mut OccupancyMetrics, + has_stream: &mut bool, ) -> error::Result> { // let mut start = Instant::now(); let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); @@ -297,7 +299,7 @@ async function run() {{ let res: any = await {main_name}(...argsArr); if (isAsyncIterable(res)) {{ for await (const chunk of res) {{ - console.log("WM_STREAM: " + chunk.replace('\n', '\\n')); + console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n')); }} res = null; }} @@ -417,6 +419,9 @@ try {{ .stderr(Stdio::piped()); start_child_process(deno_cmd, DENO_PATH.as_str(), false).await? }; + + let stream_notifier = StreamNotifier::new(conn, job); + // logs.push_str(format!("prepare: {:?}\n", start.elapsed().as_micros()).as_str()); // start = Instant::now(); let handle_result = handle_child( @@ -433,8 +438,12 @@ try {{ false, &mut Some(occupancy_metrics), None, + stream_notifier, ) .await?; + + *has_stream = handle_result.result_stream.is_some(); + // logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str()); if let Err(e) = tokio::fs::remove_dir_all(format!("{DENO_CACHE_DIR}/gen/file/{job_dir}")).await { @@ -547,6 +556,7 @@ pub async fn start_worker( None, None, None, + None, ) .await; let context_envs = build_envs_map(context.to_vec()).await; diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index aea5f7be15..e3408ffca7 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -1,6 +1,6 @@ use std::cell::RefCell; use std::env; -use std::ffi::{c_char, CString}; +use std::ffi::{c_char, CStr, CString}; use std::ptr::NonNull; use std::sync::{Arc, Mutex}; @@ -10,7 +10,7 @@ use serde_json::value::RawValue; use serde_json::{json, Value}; use uuid::Uuid; use windmill_common::error::{to_anyhow, Error, Result}; -use windmill_common::s3_helpers::S3Object; +use windmill_common::s3_helpers::{S3Object, S3_PROXY_LAST_ERRORS_CACHE}; use windmill_common::utils::sanitize_string_from_password; use windmill_common::worker::Connection; use windmill_common::workspaces::{get_ducklake_from_db_unchecked, DucklakeCatalogResourceType}; @@ -18,7 +18,7 @@ use windmill_parser_sql::{parse_duckdb_sig, parse_sql_blocks}; use windmill_queue::{CanceledBy, MiniPulledJob}; use crate::agent_workers::get_ducklake_from_agent_http; -use crate::common::{build_args_values, OccupancyMetrics}; +use crate::common::{build_args_values, get_reserved_variables, OccupancyMetrics}; use crate::handle_child::run_future_with_polling_update_job_poller; #[cfg(feature = "mysql")] use crate::mysql_executor::MysqlDatabase; @@ -37,6 +37,7 @@ pub async fn do_duckdb( // TODO #[allow(unused_variables)] column_order_ref: &mut Option>, occupancy_metrics: &mut OccupancyMetrics, + parent_runnable_path: Option, ) -> Result> { let token = client.token.clone(); let hidden_passwords = Arc::new(Mutex::new(Vec::::new())); @@ -48,7 +49,11 @@ pub async fn do_duckdb( let sig = parse_duckdb_sig(query)?.args; let mut job_args = build_args_values(job, client, conn).await?; - let (query, _) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args)?; + let reserved_variables = + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; + + let (query, _) = + &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args, &reserved_variables)?; let query = transform_s3_uris(query).await?; let job_args = { @@ -128,7 +133,7 @@ pub async fn do_duckdb( let base_internal_url = client.base_internal_url.clone(); let w_id = job.workspace_id.clone(); - let (result, column_order) = tokio::task::spawn_blocking(move || { + let result = tokio::task::spawn_blocking(move || { run_duckdb_ffi_safe( query_block_list.iter().map(String::as_str), query_block_list.len(), @@ -139,7 +144,21 @@ pub async fn do_duckdb( ) }) .await - .map_err(to_anyhow)??; + .map_err(|e| Error::from(to_anyhow(e))) + .and_then(|r| r); + let (result, column_order) = match result { + Ok(r) => r, + Err(e) => { + if let Some(s3_proxy_err) = S3_PROXY_LAST_ERRORS_CACHE.get(&client.token) { + return Err(Error::ExecutionErr(format!( + "{}\n\nS3 Related Error: {}", + e.to_string(), + s3_proxy_err, + ))); + } + return Err(e); + } + }; drop(bigquery_credentials); @@ -193,6 +212,7 @@ struct DuckDbFfiLib { column_order_ptr: *mut *mut c_char, ) -> *mut c_char, >, + free_cstr: Symbol<'static, unsafe extern "C" fn(string: *mut c_char) -> ()>, } impl DuckDbFfiLib { @@ -232,6 +252,7 @@ impl DuckDbFfiLib { let lib = Box::leak(Box::new(lib)); Ok(DuckDbFfiLib { run_duckdb_ffi: unsafe { lib.get(b"run_duckdb_ffi").map_err(to_anyhow)? }, + free_cstr: unsafe { lib.get(b"free_cstr").map_err(to_anyhow)? }, }) } } @@ -264,8 +285,9 @@ fn run_duckdb_ffi_safe<'a>( let w_id = CString::new(w_id).map_err(to_anyhow)?; let run_duckdb_ffi = &DuckDbFfiLib::get_singleton()?.run_duckdb_ffi; + let free_cstr = &DuckDbFfiLib::get_singleton()?.free_cstr; let mut column_order: *mut c_char = std::ptr::null_mut(); - let result_cstr = unsafe { + let result_str = unsafe { let ptr = run_duckdb_ffi( query_block_list.as_ptr(), query_block_list_count, @@ -275,27 +297,19 @@ fn run_duckdb_ffi_safe<'a>( w_id.as_ptr(), &mut column_order, ); - CString::from_raw(ptr) // Using from_raw to take ownership and ensure it gets freed + let str = CStr::from_ptr(ptr).to_string_lossy().to_string(); + free_cstr(ptr); + str }; let column_order = if column_order.is_null() { None } else { - Some(unsafe { - serde_json::from_str::>(&CString::from_raw(column_order).to_string_lossy())? - }) + let str = unsafe { CStr::from_ptr(column_order).to_string_lossy().to_string() }; + unsafe { free_cstr(column_order) }; + Some(serde_json::from_str::>(&str)?) }; - let result_str = result_cstr - .to_str() - .map_err(|e| { - Error::ExecutionErr(format!( - "Failed to convert result C string to Rust string: {}", - e.to_string() - )) - })? - .to_string(); - if result_str.starts_with("ERROR") { Err(Error::ExecutionErr(result_str[6..].to_string())) } else { @@ -479,6 +493,15 @@ async fn transform_attach_ducklake( let storage = ducklake.storage.storage.as_deref().unwrap_or("_default_"); let data_path = ducklake.storage.path; + // Ducklake 0.3 only requires DATA_PATH at creation and then stores it internally in the catalog + // But it will fail if DATA_PATH changes afterwards which is annoying for us + // So we always enable override + let extra_args = if extra_args.contains("OVERRIDE_DATA_PATH") { + extra_args + } else { + format!(", OVERRIDE_DATA_PATH TRUE{extra_args}") + }; + let attach_str = format!( "ATTACH 'ducklake:{db_type}:{db_conn_str}' AS {alias_name} (DATA_PATH 's3://{storage}/{data_path}'{extra_args});", ); diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 85adbf7fea..0409ceba95 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -274,6 +274,7 @@ func Run(req Req) (interface{{}}, error){{ false, &mut Some(occupation_metrics), None, + None, ) .await?; @@ -405,7 +406,10 @@ func Run(req Req) (interface{{}}, error){{ #[cfg(windows)] set_windows_env_vars(&mut run_go); - run_go.stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::piped()); + run_go + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); start_child_process(run_go, &compiled_executable_name, false).await? }; let handle_result = handle_child( @@ -422,6 +426,7 @@ func Run(req Req) (interface{{}}, error){{ false, &mut Some(occupation_metrics), None, + None, ) .await?; @@ -507,6 +512,7 @@ pub async fn install_go_dependencies( false, &mut Some(occupation_metrics), None, + None, ) .await?; @@ -622,6 +628,7 @@ pub async fn install_go_dependencies( false, &mut Some(occupation_metrics), None, + None, ) .await?; @@ -643,7 +650,7 @@ pub async fn install_go_dependencies( if non_dep_job { if let Some(db) = conn.as_sql() { sqlx::query!( - "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('5 mins')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", + "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('5 mins')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = EXCLUDED.lockfile", hash, req_content ) diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 11078c7892..0604ebff05 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -29,7 +29,7 @@ use windmill_queue::{append_logs, CanceledBy}; use std::os::unix::process::ExitStatusExt; use std::process::ExitStatus; -use std::sync::atomic::AtomicU32; +use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; use std::sync::Arc; use std::{io, panic, time::Duration}; @@ -52,7 +52,7 @@ use futures::{ stream, StreamExt, }; -use crate::common::{resolve_job_timeout, OccupancyMetrics}; +use crate::common::{resolve_job_timeout, OccupancyMetrics, StreamNotifier}; use crate::job_logger::{append_job_logs, append_result_stream, append_with_limit}; use crate::job_logger_oss::process_streaming_log_lines; use crate::worker_utils::{ping_job_status, update_worker_ping_from_job}; @@ -60,6 +60,7 @@ use crate::{MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM}; lazy_static::lazy_static! { pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true"); + pub static ref OTEL_JOB_LOGS: bool = std::env::var("OTEL_JOB_LOGS").ok().is_some_and(|x| x == "1" || x == "true"); } // - kill windows process along with all child processes @@ -114,6 +115,7 @@ pub async fn handle_child( occupancy_metrics: &mut Option<&mut OccupancyMetrics>, // Do not print logs to output, but instead save to string. pipe_stdout: Option<&mut String>, + stream_notifier: Option, ) -> error::Result { let start = Instant::now(); @@ -315,6 +317,7 @@ pub async fn handle_child( &mut rx2, child_name, &mut stream_result, + stream_notifier, ) .instrument(trace_span!("child_lines")); @@ -342,6 +345,8 @@ pub async fn handle_child( } } +pub const OTEL_PREFIX: &str = "OTEL: "; + pub async fn write_lines( output: impl stream::Stream> + Send, job_id: &Uuid, @@ -354,6 +359,7 @@ pub async fn write_lines( rx2: &mut broadcast::Receiver<()>, child_name: &str, stream_result: &mut Vec, + stream_notifier: Option, ) { let max_log_size = if *CLOUD_HOSTED { MAX_RESULT_SIZE @@ -384,6 +390,8 @@ pub async fn write_lines( let mut pipe_stdout = pipe_stdout; + let is_stream = Arc::new(AtomicBool::new(false)); + let offset = Arc::new(AtomicI32::new(0)); while let Some(line) = output.by_ref().next().await { let do_write_ = do_write.shared(); @@ -410,12 +418,18 @@ pub async fn write_lines( let job_id = job_id.clone(); let mut nstream = String::new(); + while let Some(line) = read_lines.next().await { match line { Ok(line) => { if line.is_empty() { continue; } + if *OTEL_JOB_LOGS { + if let Some(otel_suffix) = line.strip_prefix(OTEL_PREFIX) { + tracing::event!(tracing::Level::INFO, otel_suffix); + } + } if let Some(stream) = extract_stream_from_logs(&line) { let len = stream.len(); if log_remaining >= len { @@ -479,9 +493,27 @@ pub async fn write_lines( let w_id = w_id.to_string(); let job_id = job_id.clone(); let pg_log_total_size = pg_log_total_size.clone(); + let stream_notifier = stream_notifier.clone(); + let is_stream = is_stream.clone(); + let offset = offset.clone(); (do_write, write_result) = tokio::spawn(async move { if !nstream.is_empty() { - if let Err(err) = append_result_stream(&conn, &w_id, &job_id, &nstream).await { + if let Some(stream_notifier) = stream_notifier { + if !is_stream.load(Ordering::SeqCst) { + is_stream.store(true, Ordering::SeqCst); + stream_notifier.update_flow_status_with_stream_job(); + } + }; + + if let Err(err) = append_result_stream( + &conn, + &w_id, + &job_id, + &nstream, + offset.fetch_add(1, Ordering::SeqCst), + ) + .await + { tracing::error!( "Unable to send result stream for job {job_id}. Error was: {:?}", err diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index 6aa12d5024..92ef69da16 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -247,7 +247,7 @@ pub async fn resolve<'a>( if let Connection::Sql(db) = conn { sqlx::query!( - "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('5 mins')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", + "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('5 mins')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = EXCLUDED.lockfile", req_hash, lock.clone(), ) @@ -534,6 +534,7 @@ async fn compile<'a>( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -718,6 +719,7 @@ async fn run<'a>( false, &mut Some(occupancy_metrics), None, + None, ) .await?; Ok(()) diff --git a/backend/windmill-worker/src/job_logger.rs b/backend/windmill-worker/src/job_logger.rs index 263234cee9..364c352a21 100644 --- a/backend/windmill-worker/src/job_logger.rs +++ b/backend/windmill-worker/src/job_logger.rs @@ -8,6 +8,7 @@ use windmill_common::worker::{Connection, CLOUD_HOSTED}; use windmill_common::{error, DB}; use windmill_queue::append_logs; +use serde::Serialize; use std::sync::atomic::AtomicU32; use std::sync::Arc; @@ -62,22 +63,33 @@ pub async fn append_job_logs( } } +#[derive(Serialize)] +struct ResultStreamBody<'a> { + result_stream: &'a str, + offset: i32, +} + pub async fn append_result_stream( conn: &Connection, workspace_id: &str, job_id: &Uuid, nstream: &str, + offset: i32, ) -> error::Result<()> { match conn { Connection::Sql(db) => { - append_result_stream_db(db, workspace_id, job_id, nstream).await?; + append_result_stream_db(db, workspace_id, job_id, nstream, offset).await?; } Connection::Http(client) => { + let body = ResultStreamBody { result_stream: nstream, offset }; if let Err(e) = client .post::<_, String>( - &format!("/api/w/{}/agent_workers/push_logs/{}", workspace_id, job_id), + &format!( + "/api/w/{}/agent_workers/push_result_stream/{}", + workspace_id, job_id + ), None, - &nstream, + &body, ) .await { @@ -96,7 +108,7 @@ pub async fn append_logs_with_compaction( worker_name: &str, ) { let log_length = sqlx::query_scalar!( - "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, $1::text) RETURNING length(logs)", + "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, EXCLUDED.logs) RETURNING length(logs)", logs, job_id, &w_id, diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index 7f6fa8146c..c87b506686 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -43,12 +43,14 @@ use uuid::Uuid; #[cfg(feature = "deno_core")] use windmill_common::error::Error; #[cfg(feature = "deno_core")] +use windmill_common::utils::configure_client; +#[cfg(feature = "deno_core")] use windmill_common::worker::{write_file, TMP_DIR}; use windmill_common::flow_status::JobResult; use windmill_queue::CanceledBy; -use crate::common::OccupancyMetrics; +use crate::common::{OccupancyMetrics, StreamNotifier}; use windmill_common::client::AuthedClient; #[cfg(feature = "deno_core")] @@ -305,11 +307,11 @@ pub async fn eval_timeout( let mut client = authed_client.clone(); if let Some(client) = client.as_mut() { client.force_client = Some( - reqwest::ClientBuilder::new() + configure_client(reqwest::ClientBuilder::new() .user_agent("windmill/beta") .danger_accept_invalid_certs( std::env::var("ACCEPT_INVALID_CERTS").is_ok(), - ) + )) .build() .unwrap(), ); @@ -520,7 +522,7 @@ function get_from_env(name) {{ .map(|a| { format!("let {a} = get_from_env(\"{a}\");\n",) }) .join(""), if expr.contains("error") && transform_context.contains(&"previous_result".to_string()) { - "let error = previous_result.error;" + "let error = previous_result?.error;" } else { "" }, @@ -769,6 +771,8 @@ pub async fn eval_fetch_timeout( _w_id: &str, _load_client: bool, _occupation_metrics: &mut OccupancyMetrics, + _stream_notifier: Option, + _has_stream: &mut bool, ) -> anyhow::Result> { use serde_json::value::to_raw_value; Ok(to_raw_value("require deno_core").unwrap()) @@ -790,6 +794,8 @@ pub async fn eval_fetch_timeout( w_id: &str, load_client: bool, occupation_metrics: &mut OccupancyMetrics, + stream_notifier: Option, + has_stream: &mut bool, ) -> windmill_common::error::Result> { let (sender, mut receiver) = oneshot::channel::(); let (append_logs_sender, mut append_logs_receiver) = mpsc::unbounded_channel::(); @@ -806,9 +812,11 @@ pub async fn eval_fetch_timeout( let conn_ = conn.clone(); let w_id_ = w_id.to_string(); tokio::spawn(async move { + let mut offset = -1; while let Some(stream) = result_stream_receiver.recv().await { use crate::job_logger::append_result_stream; - if let Err(e) = append_result_stream(&conn_, &w_id_, &job_id, &stream).await { + offset += 1; + if let Err(e) = append_result_stream(&conn_, &w_id_, &job_id, &stream, offset).await { tracing::error!("failed to append result stream: {e}"); } } @@ -940,10 +948,18 @@ pub async fn eval_fetch_timeout( } let handle = tokio::spawn(async move { let mut result_stream = String::new(); + let mut is_stream = false; while let Some(log) = log_receiver.recv().await { use windmill_common::result_stream::extract_stream_from_logs; if let Some(stream) = extract_stream_from_logs(&log.trim_end_matches("\n")) { + if let Some(sn) = stream_notifier.as_ref() { + if !is_stream { + is_stream = true; + sn.update_flow_status_with_stream_job(); + } + } + result_stream.push_str(&stream); if let Err(e) = result_stream_sender.send(stream) { tracing::error!("failed to send result stream: {e}"); @@ -968,22 +984,24 @@ pub async fn eval_fetch_timeout( drop(js_runtime); if let Ok(r) = r { match handle.await { - Ok(Some(logs)) => Ok(merge_result_stream(r, Some(logs)).await), - Ok(None) => Ok(r), + Ok(Some(logs)) => { + Ok(merge_result_stream(r, Some(logs)).await.map(|r| (r, true))) + } + Ok(None) => Ok(r.map(|r| (r, false))), Err(e) => Err(Error::ExecutionErr(e.to_string())), } } else { - r + r.map(|r| r.map(|r| (r, false))) } // r }; let r = runtime.block_on(future)?; // tracing::info!("total: {:?}", instant.elapsed()); - r as windmill_common::error::Result> + r as windmill_common::error::Result<(Box, bool)> }); - let res = run_future_with_polling_update_job_poller( + let (res, new_has_stream) = run_future_with_polling_update_job_poller( job_id, job_timeout, conn, @@ -1002,6 +1020,7 @@ pub async fn eval_fetch_timeout( } e })?; + *has_stream = new_has_stream; *mem_peak = (res.get().len() / 1000) as i32; Ok(res) } @@ -1099,7 +1118,7 @@ function processStreamIterative(res) {{ iterator.next().then(function(result) {{ if (!result.done) {{ const chunk = result.value; - console.log("WM_STREAM: " + chunk.replace('\n', '\\n')); + console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n')); // Continue the loop step(); }} else {{ diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index ace218c207..4bf60f6d4a 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -9,6 +9,7 @@ mod agent_workers; #[cfg(feature = "python")] mod ansible_executor; mod bash_executor; +mod pwsh_executor; #[cfg(feature = "java")] mod java_executor; @@ -16,6 +17,7 @@ mod java_executor; #[cfg(feature = "ruby")] mod ruby_executor; +mod ai; mod ai_executor; mod bun_executor; pub mod common; @@ -35,6 +37,10 @@ pub mod job_logger; pub mod job_logger_ee; mod job_logger_oss; mod js_eval; +pub mod memory_common; +#[cfg(feature = "private")] +pub mod memory_ee; +pub mod memory_oss; #[cfg(feature = "mysql")] mod mysql_executor; #[cfg(feature = "nu")] @@ -56,6 +62,7 @@ pub mod result_processor; mod rust_executor; mod sanitized_sql_params; mod schema; +pub mod scoped_dependency_map; mod universal_pkg_installer; mod worker; mod worker_flow; diff --git a/backend/windmill-worker/src/memory_common.rs b/backend/windmill-worker/src/memory_common.rs new file mode 100644 index 0000000000..16821c555b --- /dev/null +++ b/backend/windmill-worker/src/memory_common.rs @@ -0,0 +1,72 @@ +use crate::ai::types::OpenAIMessage; +use std::path::PathBuf; +use tokio::{fs, io::AsyncWriteExt}; +use uuid::Uuid; +use windmill_common::worker::TMP_MEMORY_DIR; + +/// Get the file path for storing memory for a specific AI agent step +pub fn path_for(workspace_id: &str, conversation_id: Uuid, step_id: &str) -> PathBuf { + PathBuf::from(TMP_MEMORY_DIR) + .join(workspace_id) + .join(conversation_id.to_string()) + .join(format!("{step_id}.json")) +} + +/// Read messages from disk storage +pub async fn read_from_disk( + workspace_id: &str, + conversation_id: Uuid, + step_id: &str, +) -> anyhow::Result>> { + let path = path_for(workspace_id, conversation_id, step_id); + if !fs::try_exists(&path).await? { + return Ok(None); + } + + let bytes = fs::read(&path).await?; + let messages: Vec = serde_json::from_slice(&bytes)?; + Ok(Some(messages)) +} + +/// Write messages to disk storage +pub async fn write_to_disk( + workspace_id: &str, + conversation_id: Uuid, + step_id: &str, + messages: &[OpenAIMessage], +) -> anyhow::Result<()> { + let path = path_for(workspace_id, conversation_id, step_id); + + // Ensure parent directories exist + if let Some(dir) = path.parent() { + fs::create_dir_all(dir).await?; + } + + // Write atomically using a temporary file + let tmp = path.with_extension("json.tmp"); + let mut f = fs::File::create(&tmp).await?; + f.write_all(&serde_json::to_vec(messages)?).await?; + f.flush().await?; + drop(f); + + // Atomic rename + fs::rename(tmp, &path).await?; + + Ok(()) +} + +/// Delete all memory for a conversation from disk storage +pub async fn delete_conversation_from_disk( + workspace_id: &str, + conversation_id: Uuid, +) -> anyhow::Result<()> { + let conversation_path = PathBuf::from(TMP_MEMORY_DIR) + .join(workspace_id) + .join(conversation_id.to_string()); + + if fs::try_exists(&conversation_path).await? { + fs::remove_dir_all(&conversation_path).await?; + } + + Ok(()) +} diff --git a/backend/windmill-worker/src/memory_oss.rs b/backend/windmill-worker/src/memory_oss.rs new file mode 100644 index 0000000000..2444cd0ebf --- /dev/null +++ b/backend/windmill-worker/src/memory_oss.rs @@ -0,0 +1,43 @@ +#[cfg(all(feature = "private", feature = "enterprise"))] +#[allow(unused)] +pub use crate::memory_ee::*; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +use {crate::ai::types::OpenAIMessage, crate::memory_common, uuid::Uuid}; + +/// Read AI agent memory from storage +/// In OSS: always reads from disk +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn read_from_memory( + workspace_id: &str, + conversation_id: Uuid, + step_id: &str, +) -> anyhow::Result>> { + memory_common::read_from_disk(workspace_id, conversation_id, step_id).await +} + +/// Write AI agent memory to storage +/// In OSS: always writes to disk +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn write_to_memory( + workspace_id: &str, + conversation_id: Uuid, + step_id: &str, + messages: &[OpenAIMessage], +) -> anyhow::Result<()> { + if messages.is_empty() { + return Ok(()); + } + + memory_common::write_to_disk(workspace_id, conversation_id, step_id, messages).await +} + +/// Delete all memory for a conversation from storage +/// In OSS: always deletes from disk +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn delete_conversation_memory( + workspace_id: &str, + conversation_id: Uuid, +) -> anyhow::Result<()> { + memory_common::delete_conversation_from_disk(workspace_id, conversation_id).await +} diff --git a/backend/windmill-worker/src/mssql_executor.rs b/backend/windmill-worker/src/mssql_executor.rs index 7e4ccd2958..c64e644be3 100644 --- a/backend/windmill-worker/src/mssql_executor.rs +++ b/backend/windmill-worker/src/mssql_executor.rs @@ -21,7 +21,9 @@ use windmill_parser_sql::{parse_db_resource, parse_mssql_sig, parse_s3_mode}; use windmill_queue::MiniPulledJob; use windmill_queue::{append_logs, CanceledBy}; -use crate::common::{build_args_values, s3_mode_args_to_worker_data, OccupancyMetrics}; +use crate::common::{ + build_args_values, get_reserved_variables, s3_mode_args_to_worker_data, OccupancyMetrics, +}; use crate::handle_child::run_future_with_polling_update_job_poller; use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; use windmill_common::client::AuthedClient; @@ -56,7 +58,7 @@ lazy_static::lazy_static! { pub async fn do_mssql( job: &MiniPulledJob, - client: &AuthedClient, + authed_client: &AuthedClient, query: &str, conn: &Connection, mem_peak: &mut i32, @@ -64,15 +66,17 @@ pub async fn do_mssql( worker_name: &str, occupancy_metrics: &mut OccupancyMetrics, job_dir: &str, + parent_runnable_path: Option, ) -> error::Result> { - let mssql_args = build_args_values(job, client, conn).await?; + let mssql_args = build_args_values(job, authed_client, conn).await?; let inline_db_res_path = parse_db_resource(&query); - let s3 = parse_s3_mode(&query)?.map(|s3| s3_mode_args_to_worker_data(s3, client.clone(), job)); + let s3 = parse_s3_mode(&query)? + .map(|s3| s3_mode_args_to_worker_data(s3, authed_client.clone(), job)); let db_arg = if let Some(inline_db_res_path) = inline_db_res_path { Some( - client + authed_client .get_resource_value_interpolated::( &inline_db_res_path, Some(job.id.to_string()), @@ -189,8 +193,11 @@ pub async fn do_mssql( .map_err(|x| Error::ExecutionErr(x.to_string()))? .args; + let reserved_variables = + get_reserved_variables(job, &authed_client.token, conn, parent_runnable_path).await?; + let (query, args_to_skip) = - &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &mssql_args)?; + &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &mssql_args, &reserved_variables)?; let mut prepared_query = Query::new(query.to_owned()); for arg in &sig { diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index 04403e4e08..55cbe35893 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -26,7 +26,10 @@ use windmill_queue::CanceledBy; use windmill_queue::MiniPulledJob; use crate::{ - common::{build_args_values, s3_mode_args_to_worker_data, OccupancyMetrics, S3ModeWorkerData}, + common::{ + build_args_values, get_reserved_variables, s3_mode_args_to_worker_data, OccupancyMetrics, + S3ModeWorkerData, + }, handle_child::run_future_with_polling_update_job_poller, sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args, }; @@ -149,6 +152,7 @@ pub async fn do_mysql( worker_name: &str, column_order: &mut Option>, occupancy_metrics: &mut OccupancyMetrics, + parent_runnable_path: Option, ) -> windmill_common::error::Result> { let job_args = build_args_values(job, client, conn).await?; @@ -198,7 +202,11 @@ pub async fn do_mysql( .map_err(|x| Error::ExecutionErr(x.to_string()))? .args; - let (query, args_to_skip) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args)?; + let reserved_variables = + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; + + let (query, args_to_skip) = + &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args, &reserved_variables)?; let using_named_params = RE_ARG_MYSQL_NAMED.captures_iter(query).count() > 0; diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index a52270e745..2497757d9d 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -340,6 +340,7 @@ async fn run<'a>( false, &mut Some(occupancy_metrics), None, + None, ) .await?; Ok(()) diff --git a/backend/windmill-worker/src/oracledb_executor.rs b/backend/windmill-worker/src/oracledb_executor.rs index 244cbb1958..a1c20d1282 100644 --- a/backend/windmill-worker/src/oracledb_executor.rs +++ b/backend/windmill-worker/src/oracledb_executor.rs @@ -23,11 +23,11 @@ use windmill_queue::CanceledBy; use crate::{ common::{ - build_args_values, check_executor_binary_exists, s3_mode_args_to_worker_data, - OccupancyMetrics, S3ModeWorkerData, + build_args_values, check_executor_binary_exists, get_reserved_variables, + s3_mode_args_to_worker_data, OccupancyMetrics, S3ModeWorkerData, }, handle_child::run_future_with_polling_update_job_poller, - sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args + sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args, }; use windmill_common::client::AuthedClient; @@ -343,6 +343,7 @@ pub async fn do_oracledb( worker_name: &str, column_order: &mut Option>, occupancy_metrics: &mut OccupancyMetrics, + parent_runnable_path: Option, ) -> windmill_common::error::Result> { check_executor_binary_exists( "the Oracle client lib", @@ -381,7 +382,11 @@ pub async fn do_oracledb( .map_err(|x| Error::ExecutionErr(x.to_string()))? .args; - let (query, args_to_skip) = sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args)?; + let reserved_variables = + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; + + let (query, args_to_skip) = + sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args, &reserved_variables)?; let (statement_values, errors) = get_statement_values(sig.clone(), &job_args, &args_to_skip); diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 2c7fe123dc..9bb7c70cda 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -37,7 +37,8 @@ use windmill_parser_sql::{ use windmill_queue::{CanceledBy, MiniPulledJob}; use crate::common::{ - build_args_values, s3_mode_args_to_worker_data, sizeof_val, OccupancyMetrics, S3ModeWorkerData, + build_args_values, get_reserved_variables, s3_mode_args_to_worker_data, sizeof_val, + OccupancyMetrics, S3ModeWorkerData, }; use crate::handle_child::run_future_with_polling_update_job_poller; use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; @@ -88,7 +89,7 @@ fn do_postgresql_inner<'a>( let arg_t = arg .otyp .as_ref() - .ok_or_else(|| anyhow::anyhow!("Missing otzyp for pg arg"))?; + .ok_or_else(|| anyhow::anyhow!("Missing otyp for pg arg"))?; let typ = &arg.typ; let param = convert_val(value, arg_t, typ)?; query_params.push(param); @@ -186,6 +187,7 @@ pub async fn do_postgresql( worker_name: &str, column_order: &mut Option>, occupancy_metrics: &mut OccupancyMetrics, + parent_runnable_path: Option, ) -> error::Result> { let pg_args = build_args_values(job, client, conn).await?; @@ -312,7 +314,11 @@ pub async fn do_postgresql( let sig = parse_pgsql_sig(&query).map_err(|x| Error::ExecutionErr(x.to_string()))?; - let (query, _) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig.args, &pg_args)?; + let reserved_variables = + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; + + let (query, _) = + &sanitize_and_interpolate_unsafe_sql_args(query, &sig.args, &pg_args, &reserved_variables)?; let queries = parse_sql_blocks(query); @@ -555,12 +561,16 @@ fn convert_vec_val( chrono::NaiveTime::parse_from_str(x, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default() }) })?)), - "timestamp" | "timestamptz" => Ok(Box::new(map_as_single_type(vec, |v| { + "timestamp" => Ok(Box::new(map_as_single_type(vec, |v| { v.as_str().map(|x| { chrono::NaiveDateTime::parse_from_str(x, "%Y-%m-%dT%H:%M:%S.%3fZ") .unwrap_or_default() }) })?)), + "timestamptz" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str() + .map(|x| x.parse::>().unwrap_or_default()) + })?)), "jsonb" | "json" => Ok(Box::new( vec.map(|v| v.clone().into_iter().map(Some).collect_vec()), )), @@ -605,7 +615,8 @@ fn convert_val( "uuid" => Ok(Box::new(None::)), "date" => Ok(Box::new(None::)), "time" | "timetz" => Ok(Box::new(None::)), - "timestamp" | "timestamptz" => Ok(Box::new(None::)), + "timestamp" => Ok(Box::new(None::)), + "timestamptz" => Ok(Box::new(None::>)), "jsonb" | "json" => Ok(Box::new(None::>)), "bytea" => Ok(Box::new(None::>)), "text" | "varchar" => Ok(Box::new(None::)), @@ -668,17 +679,22 @@ fn convert_val( chrono::NaiveTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default(); Ok(Box::new(time)) } - Value::String(s) if arg_t == "timestamp" || arg_t == "timestamptz" => { + Value::String(s) if arg_t == "timestamp" => { let datetime = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ") .unwrap_or_default(); Ok(Box::new(datetime)) } + Value::String(s) if arg_t == "timestamptz" => { + let datetime = s.parse::>().unwrap_or_default(); + Ok(Box::new(datetime)) + } Value::String(s) if arg_t == "bytea" => { let bytes = engine::general_purpose::STANDARD .decode(s) .unwrap_or(vec![]); Ok(Box::new(bytes)) } + Value::Array(_) if arg_t == "jsonb" || arg_t == "json" => Ok(Box::new(value.clone())), Value::Object(_) if arg_t == "text" || arg_t == "varchar" => { Ok(Box::new(serde_json::to_string(value).map_err(|err| { Error::ExecutionErr(format!("Failed to convert JSON to text: {}", err)) diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index cdfd4cfe6f..b272d631da 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -106,6 +106,7 @@ pub async fn composer_install( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -343,6 +344,7 @@ try {{ false, &mut Some(occupancy_metrics), None, + None, ) .await?; read_result(job_dir, None).await diff --git a/backend/windmill-worker/src/pwsh_executor.rs b/backend/windmill-worker/src/pwsh_executor.rs new file mode 100644 index 0000000000..cd3161ef12 --- /dev/null +++ b/backend/windmill-worker/src/pwsh_executor.rs @@ -0,0 +1,623 @@ +use std::{collections::HashMap, fs, process::Stdio}; + +use regex::Regex; +use serde_json::{json, value::RawValue}; +use sqlx::types::Json; +use tokio::process::Command; +use windmill_common::client::AuthedClient; +use windmill_common::error::Error; +use windmill_common::worker::{to_raw_value, write_file, Connection}; +use windmill_queue::{ + append_logs, CanceledBy, MiniPulledJob, INIT_SCRIPT_PATH_PREFIX, PERIODIC_SCRIPT_PATH_PREFIX, +}; + +#[cfg(windows)] +use crate::SYSTEM_ROOT; + +const NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT: &str = + include_str!("../nsjail/run.powershell.config.proto"); + +lazy_static::lazy_static! { + static ref RE_POWERSHELL_IMPORTS: Regex = Regex::new(r#"^Import-Module\s+(?:-Name\s+)?"?([^\s"]+)"?(?:\s+-RequiredVersion\s+"?([^\s"]+)"?)?"#).unwrap(); +} + +use crate::{ + common::{ + build_args_map, get_reserved_variables, read_file, read_file_content, start_child_process, + OccupancyMetrics, + }, + handle_child::handle_child, + DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, POWERSHELL_CACHE_DIR, + POWERSHELL_PATH, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, PROXY_ENVS, TZ_ENV, +}; + +fn val_to_pwsh_param(v: serde_json::Value) -> String { + match v { + serde_json::Value::Array(x) => format!( + "@({})", + x.into_iter() + .map(|v| val_to_pwsh_param(v)) + .collect::>() + .join(",") + ), + serde_json::Value::Object(x) => { + let str = serde_json::to_string(&x).unwrap_or_else(|_| "{}".to_string()); + let escaped = str.replace("'", "''"); + format!("(ConvertFrom-Json '{escaped}')") + } + serde_json::Value::Null => "$null".to_string(), + serde_json::Value::Bool(x) => format!("${x}"), + serde_json::Value::String(x) => { + let escaped = x.replace("'", "''"); + format!("'{escaped}'") + } + serde_json::Value::Number(x) => x.to_string(), + } +} + +fn raw_to_pwsh_param(x: &str) -> String { + match serde_json::from_str::(x) { + Ok(v) => val_to_pwsh_param(v), + Err(e) => { + tracing::error!("Error converting JSON to string: {:?}", e); + "$null".to_string() + } + } +} + +fn generate_powershell_install_code() -> String { + r#" +$ErrorActionPreference = 'Stop' +$availableModules = Get-Module -ListAvailable +$path = '{path}' +$hasPrivateRepo = {has_private_repo} +$jobId = '{job_id}' +$privateRepoUrl = '{private_repo_url}' +$privateRepoPat = '{private_repo_pat}' + +# Setup private repository if configured +$repoName = $null +$credentials = $null +if ($hasPrivateRepo) { + $repoName = "windmill-private-$jobId" + $repoUri = "$privateRepoUrl" + + # Create PSCredential for authentication + $username = "token" + $patToken = ConvertTo-SecureString $privateRepoPat -AsPlainText -Force + $credentials = New-Object System.Management.Automation.PSCredential($username, $patToken) + + Write-Host "Registering temporary repository: $repoName" + + # Remove repository if it already exists + Unregister-PSResourceRepository -Name $repoName -ErrorAction SilentlyContinue + Register-PSResourceRepository -Name $repoName -Uri $repoUri -Trusted +} + +try { + $moduleRequests = @({modules}) + foreach ($moduleRequest in $moduleRequests) { + $moduleName = $moduleRequest.Name + $requiredVersion = $moduleRequest.Version + + # Check if module is already installed with the required version (case-insensitive) + $isInstalled = $false + if ($requiredVersion) { + $isInstalled = $availableModules | Where-Object { $_.Name -eq $moduleName -and $_.Version -eq $requiredVersion } + } else { + $isInstalled = $availableModules | Where-Object { $_.Name -eq $moduleName } + } + + if (-not $isInstalled) { + $moduleFound = $false + + # First try private repository if configured + if ($hasPrivateRepo) { + $findParams = @{ Name = $moduleName; Repository = $repoName; ErrorAction = 'SilentlyContinue'; Credential = $credentials } + if ($requiredVersion) { $findParams.Version = $requiredVersion } + + $privateModule = Find-PSResource @findParams + if ($privateModule) { + $moduleFound = $true + $versionInfo = if ($requiredVersion) { " version $requiredVersion" } else { "" } + Write-Host "Found module $moduleName$versionInfo in private repository, installing from there..." + + $saveParams = @{ Name = $moduleName; Path = $path; Repository = $repoName; Credential = $credentials } + if ($requiredVersion) { $saveParams.Version = $requiredVersion } + Save-PSResource @saveParams + } + } + + # If not found in private repo (or no private repo configured), try all repositories + if (-not $moduleFound) { + $versionInfo = if ($requiredVersion) { " version $requiredVersion" } else { "" } + Write-Host "Installing module $moduleName$versionInfo from public repositories..." + + $saveParams = @{ Name = $moduleName; Path = $path; TrustRepository = $true } + if ($requiredVersion) { $saveParams.Version = $requiredVersion } + Save-PSResource @saveParams + } + } else { + $versionInfo = if ($requiredVersion) { " version $requiredVersion" } else { "" } + Write-Host "Module $moduleName$versionInfo already installed" + } + } +} finally { + if ($hasPrivateRepo) { + Write-Host "Unregistering temporary repository: $repoName" + Unregister-PSResourceRepository -Name $repoName + } +} +"#.to_string() +} + +async fn scan_module_directories() -> Result, Error> { + let mut module_dirs = HashMap::new(); + let cache_dir = std::path::Path::new(POWERSHELL_CACHE_DIR); + + if let Ok(entries) = fs::read_dir(cache_dir) { + for entry in entries { + if let Ok(entry) = entry { + let module_path = entry.path(); + if module_path.is_dir() { + if let Some(module_name) = module_path.file_name().and_then(|n| n.to_str()) { + module_dirs.insert( + module_name.to_lowercase(), // Use lowercase for case-insensitive lookup + module_path.to_string_lossy().to_string(), + ); + } + } + } + } + } + + Ok(module_dirs) +} + +async fn get_module_versions(module_path: &str) -> Result, Error> { + let mut versions = Vec::new(); + + // Look for version subdirectories within the module directory + if let Ok(version_entries) = fs::read_dir(module_path) { + for version_entry in version_entries { + if let Ok(version_entry) = version_entry { + let version_path = version_entry.path(); + if version_path.is_dir() { + let version = version_path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or_default() + .to_string(); + + // Check if this looks like a version (contains dots and numbers) + if version.chars().any(|c| c.is_numeric()) && version.contains('.') { + versions.push(version); + } + } + } + } + } + + // If no version subdirectories found, treat as single version installation + if versions.is_empty() { + versions.push("unknown".to_string()); + } + + Ok(versions) +} + +async fn check_module_installed( + module_dirs: &HashMap, + module_name: &str, + required_version: Option<&str>, +) -> Result<(bool, Vec), Error> { + let module_key = module_name.to_lowercase(); + + if let Some(module_path) = module_dirs.get(&module_key) { + let versions = get_module_versions(module_path).await?; + let is_installed = match required_version { + Some(version) => versions.iter().any(|v| v == version), + None => !versions.is_empty(), + }; + Ok((is_installed, versions)) + } else { + Ok((false, Vec::new())) + } +} + +#[derive(Debug)] +struct ModuleRequest { + name: String, + version: Option, +} + +#[tracing::instrument(level = "trace", skip_all)] +pub async fn handle_powershell_job( + mem_peak: &mut i32, + canceled_by: &mut Option, + job: &MiniPulledJob, + db: &Connection, + client: &AuthedClient, + parent_runnable_path: Option, + content: &str, + job_dir: &str, + shared_mount: &str, + base_internal_url: &str, + worker_name: &str, + envs: HashMap, + occupancy_metrics: &mut OccupancyMetrics, +) -> Result, Error> { + let pwsh_args = { + let args = build_args_map(job, client, &db).await?.map(Json); + let job_args = if args.is_some() { + args.as_ref() + } else { + job.args.as_ref() + }; + + let args_owned = windmill_parser_bash::parse_powershell_sig(&content)? + .args + .iter() + .map(|arg| { + ( + arg.name.clone(), + job_args.and_then(|x| x.get(&arg.name).map(|x| raw_to_pwsh_param(x.get()))), + ) + }) + .collect::)>>(); + + args_owned + .into_iter() + .filter_map(|(n, v)| v.map(|v| format!("-{n} {v}"))) + .collect::>() + .join(" ") + }; + + // First, collect all imported modules + let mut imported_modules: Vec<(String, Option)> = Vec::new(); + for line in content.lines() { + for cap in RE_POWERSHELL_IMPORTS.captures_iter(line) { + let module_name = cap.get(1).unwrap().as_str().to_string(); + let required_version = cap.get(2).map(|m| m.as_str().to_string()); + imported_modules.push((module_name, required_version)); + } + } + + // Only scan the top-level cache directory if there are modules to check + let module_dirs = if !imported_modules.is_empty() { + scan_module_directories().await? + } else { + HashMap::new() + }; + + let mut modules_to_install: Vec = Vec::new(); + let mut logs1 = String::new(); + + for (module_name, required_version) in imported_modules { + // Check if this specific module is already installed, only scanning versions if needed + let (is_installed, installed_versions) = + check_module_installed(&module_dirs, &module_name, required_version.as_deref()).await?; + + if !is_installed { + modules_to_install.push(ModuleRequest { + name: module_name.clone(), + version: required_version.clone(), + }); + } else { + // Log what versions are actually installed + let version_info = if let Some(version) = &required_version { + format!(" version {} found in cache", version) + } else if installed_versions.len() == 1 { + format!(" (version {}) found in cache", installed_versions[0]) + } else if installed_versions.len() > 1 { + format!( + " (versions: {}) found in cache", + installed_versions.join(", ") + ) + } else { + " found in cache".to_string() + }; + logs1.push_str(&format!("\n{}{}", module_name, version_info)); + } + } + + if !logs1.is_empty() { + append_logs(&job.id, &job.workspace_id, logs1, db).await; + } + + if !modules_to_install.is_empty() { + let powershell_repo_url = POWERSHELL_REPO_URL.read().await.clone(); + let powershell_repo_pat = POWERSHELL_REPO_PAT.read().await.clone(); + let has_private_repo = powershell_repo_url.is_some() && powershell_repo_pat.is_some(); + + let modules_list = modules_to_install + .iter() + .map(|module_req| { + if let Some(version) = &module_req.version { + format!( + "@{{ Name = '{}'; Version = '{}' }}", + module_req.name, version + ) + } else { + format!("@{{ Name = '{}'; Version = $null }}", module_req.name) + } + }) + .collect::>() + .join(", "); + + let install_string = generate_powershell_install_code() + .replace("{path}", POWERSHELL_CACHE_DIR) + .replace("{job_id}", &job.id.to_string()) + .replace("{has_private_repo}", &format!("${has_private_repo}")) + .replace( + "{private_repo_url}", + &powershell_repo_url.unwrap_or_default(), + ) + .replace( + "{private_repo_pat}", + &powershell_repo_pat.unwrap_or_default(), + ) + .replace("{modules}", &modules_list); + let mut cmd = Command::new(POWERSHELL_PATH.as_str()); + cmd.args(&["-Command", &install_string]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let child = start_child_process(cmd, POWERSHELL_PATH.as_str(), false).await?; + + handle_child( + &job.id, + db, + mem_peak, + canceled_by, + child, + false, + worker_name, + &job.workspace_id, + "powershell install", + job.timeout, + false, + &mut Some(occupancy_metrics), + None, + None, + ) + .await?; + } + + let mut logs2 = "".to_string(); + logs2.push_str("\n\n--- POWERSHELL CODE EXECUTION ---\n"); + append_logs(&job.id, &job.workspace_id, logs2, db).await; + + // make sure default (only allhostsallusers) modules are loaded, disable autoload (cache can be large to explore especially on cloud) and add /tmp/windmill/cache to PSModulePath + #[cfg(unix)] + let profile = format!( + "$PSModuleAutoloadingPreference = 'None' +$PSModulePathBackup = $env:PSModulePath +$env:PSModulePath = \"$PSHome/Modules\" +Get-Module -ListAvailable | Import-Module +$env:PSModulePath = \"{}:$PSModulePathBackup\"", + POWERSHELL_CACHE_DIR + ); + + #[cfg(windows)] + let profile = format!( + "$PSModuleAutoloadingPreference = 'None' +$PSModulePathBackup = $env:PSModulePath +$env:PSModulePath = \"C:\\Program Files\\PowerShell\\7\\Modules\" +Get-Module -ListAvailable | Import-Module +$env:PSModulePath = \"{};$PSModulePathBackup\"", + POWERSHELL_CACHE_DIR + ); + + // NOTE: powershell error handling / termination is quite tricky compared to bash + // here we're trying to catch terminating errors and propagate the exit code + // to the caller such that the job will be marked as failed. It's up to the user + // to catch specific errors in their script not caught by the below as there is no + // generic set -eu as in bash + let strict_termination_start = "$ErrorActionPreference = 'Stop'\n\ + Set-StrictMode -Version Latest\n\ + try {\n"; + + let strict_termination_end = "\n\ + } catch {\n\ + Write-Output \"An error occurred:\n\"\ + Write-Output $_ + exit 1\n\ + }\n"; + + // make sure param() is first + let param_match = windmill_parser_bash::RE_POWERSHELL_PARAM.find(&content); + let content: String = if let Some(param_match) = param_match { + let param_match = param_match.as_str(); + format!( + "{}\n{}\n{}\n{}\n{}", + param_match, + profile, + strict_termination_start, + content.replace(param_match, ""), + strict_termination_end + ) + } else { + format!("{}\n{}", profile, content) + }; + + write_file(job_dir, "main.ps1", content.as_str())?; + + write_file( + job_dir, + "wrapper.ps1", + &format!( + "$ErrorActionPreference = 'Stop'\n\ + $pipe = New-TemporaryFile\n\ + ./main.ps1 {pwsh_args} 2>&1 | Tee-Object -FilePath $pipe\n\ + Get-Content -Path $pipe | Select-Object -Last 1 | Set-Content -Path './result2.out'\n\ + Remove-Item $pipe\n\ + exit $LASTEXITCODE\n" + ), + )?; + + let mut reserved_variables = + get_reserved_variables(job, &client.token, db, parent_runnable_path).await?; + reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); + + let _ = write_file(job_dir, "result.json", "")?; + let _ = write_file(job_dir, "result.out", "")?; + let _ = write_file(job_dir, "result2.out", "")?; + + let nsjail = !*DISABLE_NSJAIL + && job + .runnable_path + .as_ref() + .map(|x| { + !x.starts_with(INIT_SCRIPT_PATH_PREFIX) + && !x.starts_with(PERIODIC_SCRIPT_PATH_PREFIX) + }) + .unwrap_or(true); + let child = if nsjail { + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{SHARED_MOUNT}", shared_mount) + .replace("{CACHE_DIR}", POWERSHELL_CACHE_DIR), + )?; + let cmd_args = vec![ + "--config", + "run.config.proto", + "--", + POWERSHELL_PATH.as_str(), + "wrapper.ps1", + ]; + let mut cmd = Command::new(NSJAIL_PATH.as_str()); + cmd.current_dir(job_dir) + .env_clear() + .envs(PROXY_ENVS.clone()) + .envs(reserved_variables) + .env("TZ", TZ_ENV.as_str()) + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .args(cmd_args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + start_child_process(cmd, NSJAIL_PATH.as_str(), false).await? + } else { + let mut cmd = Command::new(POWERSHELL_PATH.as_str()); + let cmd_args; + + #[cfg(unix)] + { + cmd_args = vec!["wrapper.ps1"]; + } + + #[cfg(windows)] + { + cmd_args = vec![r".\wrapper.ps1"]; + } + + cmd.current_dir(job_dir) + .env_clear() + .envs(envs) + .envs(reserved_variables) + .env("TZ", TZ_ENV.as_str()) + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("HOME", HOME_ENV.as_str()) + .args(&cmd_args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + cmd.env("SystemRoot", SYSTEM_ROOT.as_str()) + .env("WINDIR", SYSTEM_ROOT.as_str()) + .env( + "LOCALAPPDATA", + std::env::var("LOCALAPPDATA") + .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())), + ) + .env( + "ProgramData", + std::env::var("ProgramData") + .unwrap_or_else(|_| String::from("C:\\ProgramData")), + ) + .env( + "ProgramFiles", + std::env::var("ProgramFiles") + .unwrap_or_else(|_| String::from("C:\\Program Files")), + ) + .env( + "ProgramFiles(x86)", + std::env::var("ProgramFiles(x86)") + .unwrap_or_else(|_| String::from("C:\\Program Files (x86)")), + ) + .env( + "ProgramW6432", + std::env::var("ProgramW6432") + .unwrap_or_else(|_| String::from("C:\\Program Files")), + ) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ) + .env( + "PATHEXT", + std::env::var("PATHEXT").unwrap_or_else(|_| { + String::from(".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL") + }), + ) + .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + } + + start_child_process(cmd, POWERSHELL_PATH.as_str(), false).await? + }; + + handle_child( + &job.id, + db, + mem_peak, + canceled_by, + child, + !*DISABLE_NSJAIL, + worker_name, + &job.workspace_id, + "powershell run", + job.timeout, + false, + &mut Some(occupancy_metrics), + None, + None, + ) + .await?; + + let result_json_path = format!("{job_dir}/result.json"); + if let Ok(metadata) = tokio::fs::metadata(&result_json_path).await { + if metadata.len() > 0 { + return Ok(read_file(&result_json_path).await?); + } + } + + let result_out_path = format!("{job_dir}/result.out"); + if let Ok(metadata) = tokio::fs::metadata(&result_out_path).await { + if metadata.len() > 0 { + let result = read_file_content(&result_out_path).await?; + return Ok(to_raw_value(&json!(result))); + } + } + + let result_out_path2 = format!("{job_dir}/result2.out"); + if tokio::fs::metadata(&result_out_path2).await.is_ok() { + let result = read_file_content(&result_out_path2) + .await? + .trim() + .to_string(); + return Ok(to_raw_value(&json!(result))); + } + + Ok(to_raw_value(&json!( + "No result.out, result2.out or result.json found" + ))) +} diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index c41089652b..7ae80a97b6 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -121,7 +121,7 @@ use windmill_common::s3_helpers::OBJECT_STORE_SETTINGS; use crate::{ common::{ create_args_and_out_file, get_reserved_variables, read_file, read_result, - start_child_process, OccupancyMetrics, + start_child_process, OccupancyMetrics, StreamNotifier, }, handle_child::handle_child, worker_utils::ping_job_status, @@ -386,6 +386,7 @@ pub async fn uv_pip_compile( false, occupancy_metrics, None, + None, ) .await .map_err(|e| { @@ -412,7 +413,7 @@ pub async fn uv_pip_compile( ); if let Some(db) = conn.as_sql() { sqlx::query!( - "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('5 mins')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", + "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('5 mins')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = EXCLUDED.lockfile", req_hash, lockfile ).fetch_optional(db).await?; @@ -543,7 +544,9 @@ pub async fn handle_python_job( new_args: &mut Option>>, occupancy_metrics: &mut OccupancyMetrics, precomputed_agent_info: Option, + has_stream: &mut bool, ) -> windmill_common::error::Result> { + let script_path = crate::common::use_flow_root_path(job.runnable_path()); let annotations = PythonAnnotations::parse(inner_content); @@ -864,6 +867,8 @@ mount {{ start_child_process(python_cmd, &python_path, false).await? }; + let stream_notifier = StreamNotifier::new(conn, job); + let handle_result = handle_child( &job.id, conn, @@ -878,9 +883,12 @@ mount {{ false, &mut Some(occupancy_metrics), None, + stream_notifier, ) .await?; + *has_stream = handle_result.result_stream.is_some(); + if apply_preprocessor { let args = read_file(&format!("{job_dir}/args.json")) .await @@ -1174,13 +1182,13 @@ async fn handle_python_deps( let (v, requirements_lines, error_hint) = match conn { Connection::Sql(db) => { let mut version_specifiers = vec![]; - let (r, h) = windmill_parser_py_imports::parse_python_imports( + let (r, h) = Box::pin(windmill_parser_py_imports::parse_python_imports( inner_content, w_id, script_path, db, &mut version_specifiers, - ) + )) .await?; let v = PyV::resolve( @@ -1860,7 +1868,7 @@ pub async fn handle_python_reqs( if let Err(e) = pull { tracing::info!( workspace_id = %w_id, - "No tarball was found for {venv_p} on S3 or different problem occured {job_id}:\n{e}", + "No tarball was found for {venv_p} on S3 or different problem occurred {job_id}:\n{e}", ); } else { print_success( @@ -1920,12 +1928,21 @@ pub async fn handle_python_reqs( } }; - let mut stderr_buf = String::new(); - let mut stderr_pipe = uv_install_proccess - .stderr() - .take() - .ok_or(anyhow!("Cannot take stderr from uv_install_proccess"))?; - let stderr_future = stderr_pipe.read_to_string(&mut stderr_buf); + let (mut stderr_buf, mut stdout_buf) = Default::default(); + let (mut stderr_pipe, mut stdout_pipe) = ( + uv_install_proccess + .stderr() + .take() + .ok_or(anyhow!("Cannot take stderr from uv_install_proccess"))?, + uv_install_proccess + .stdout() + .take() + .ok_or(anyhow!("Cannot take stdout from uv_install_proccess"))? + ); + let (stderr_future, stdout_future) = ( + stderr_pipe.read_to_string(&mut stderr_buf), + stdout_pipe.read_to_string(&mut stdout_buf) + ); if let Some(pid) = pids.lock().await.get_mut(i) { *pid = uv_install_proccess.id(); @@ -1943,25 +1960,27 @@ pub async fn handle_python_reqs( pids.lock().await.get_mut(i).and_then(|e| e.take()); return Err(anyhow::anyhow!("uv pip install was canceled")); }, - (_, exitstatus) = async { + (_, _, exitstatus) = async { // See tokio::process::Child::wait_with_output() for more context // Sometimes uv_install_proccess.wait() is not exiting if stderr is not awaited before it :/ - (stderr_future.await, Box::into_pin(uv_install_proccess.wait()).await) + (stderr_future.await, stdout_future.await, Box::into_pin(uv_install_proccess.wait()).await) } => match exitstatus { Ok(status) => if !status.success() { + let code = status.code(); tracing::warn!( workspace_id = %w_id, "uv install {} did not succeed, exit status: {:?}", &req, - status.code() + code ); append_logs( &job_id, w_id, format!( - "\nError while installing {}:\n{stderr_buf}", - &req + "\nError while installing {}: \nStderr:\n{stderr_buf}\nStdout:\n{stdout_buf}\nExit status: {:?}", + &req, + code ), &conn, ) @@ -2050,6 +2069,13 @@ pub async fn handle_python_reqs( .unwrap_or(Err(anyhow!("Problem by joining handle"))) { failed = true; + append_logs( + &job_id, + w_id, + format!("\nEnv installation failed: {:?}", e), + conn, + ) + .await; tracing::warn!( workspace_id = %w_id, "Env installation failed: {:?}", @@ -2148,6 +2174,7 @@ pub async fn start_worker( None, None, None, + None, ) .await .to_vec(); @@ -2269,6 +2296,7 @@ for line in sys.stdin: None, None, None, + None, ) .await; diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index 3d71e9f9ec..2c5b828313 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -655,6 +655,7 @@ impl PyV { false, occupancy_metrics, None, + None, ) .await?; Ok(()) diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 591d6bf11f..7cfe63dc34 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -19,6 +19,7 @@ use uuid::Uuid; use windmill_common::{ add_time, error::{self, Error}, + flow_status::FlowJobDuration, jobs::JobKind, utils::WarnAfterExt, worker::{to_raw_value, Connection, WORKER_GROUP}, @@ -30,8 +31,7 @@ use windmill_common::{ use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{ - append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob, ValidableJson, - WrappedError, INIT_SCRIPT_TAG, + CanceledBy, INIT_SCRIPT_TAG, JobCompleted, MiniCompletedJob, MiniPulledJob, ValidableJson, WrappedError, append_logs, get_mini_completed_job }; use serde_json::{json, value::RawValue, Value}; @@ -342,6 +342,7 @@ pub fn start_background_processor( &w_id, success, Arc::new(result), + None, true, &same_worker_tx, &worker_dir, @@ -398,7 +399,7 @@ async fn send_job_completed(job_completed_tx: JobCompletedSender, jc: JobComplet } pub async fn process_result( - job: Arc, + job: MiniCompletedJob, result: error::Result>>, job_dir: &str, job_completed_tx: JobCompletedSender, @@ -410,6 +411,7 @@ pub async fn process_result( preprocessed_args: Option>>, conn: &Connection, duration: Option, + has_stream: bool, ) -> error::Result { match result { Ok(result) => { @@ -426,6 +428,8 @@ pub async fn process_result( cached_res_path, token: token.to_string(), duration, + has_stream: Some(has_stream), + from_cache: None, }, ) .with_context(windmill_common::otel_oss::otel_ctx()) @@ -488,6 +492,8 @@ pub async fn process_result( cached_res_path, token: token.to_string(), duration, + has_stream: Some(has_stream), + from_cache: None, }, ) .with_context(windmill_common::otel_oss::otel_ctx()) @@ -532,7 +538,7 @@ pub async fn handle_receive_completed_job( handle_job_error( db, &client, - job.as_ref(), + &job, mem_peak, canceled_by, err, @@ -562,6 +568,8 @@ pub async fn process_completed_job( duration, result_columns, preprocessed_args, + has_stream, + from_cache, .. }: JobCompleted, client: &AuthedClient, @@ -582,6 +590,7 @@ pub async fn process_completed_job( let parent_job = job.parent_job.clone(); let job_id = job.id.clone(); let workspace_id = job.workspace_id.clone(); + let started_at = job.started_at.clone(); if job.flow_step_id.as_deref() == Some("preprocessor") { // Do this before inserting to `v2_job_completed` for backwards compatibility @@ -614,7 +623,7 @@ pub async fn process_completed_job( add_time!(bench, "pre add_completed_job"); - add_completed_job( + let (_, duration) = add_completed_job( db, &job, true, @@ -625,6 +634,8 @@ pub async fn process_completed_job( canceled_by, false, duration, + has_stream.unwrap_or(false), + from_cache.unwrap_or(false), ) .await?; drop(job); @@ -642,6 +653,7 @@ pub async fn process_completed_job( &workspace_id, true, result, + started_at.map(|x| FlowJobDuration { started_at: x, duration_ms: duration }), false, &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(), &worker_dir, @@ -682,6 +694,12 @@ pub async fn process_completed_job( &job.workspace_id, false, Arc::new(serde_json::value::to_raw_value(&result).unwrap()), + duration.and_then(|d| { + job.started_at.map(|started_at| FlowJobDuration { + started_at: started_at, + duration_ms: d, + }) + }), false, &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(), &worker_dir, @@ -702,7 +720,7 @@ pub async fn process_completed_job( pub async fn handle_non_flow_job_error( db: &DB, - job: &MiniPulledJob, + job: &MiniCompletedJob, mem_peak: i32, canceled_by: Option, err_string: String, @@ -733,7 +751,7 @@ pub async fn handle_non_flow_job_error( pub async fn handle_job_error( db: &DB, client: &AuthedClient, - job: &MiniPulledJob, + job: &MiniCompletedJob, mem_peak: i32, canceled_by: Option, err: Error, @@ -783,6 +801,7 @@ pub async fn handle_job_error( &job.workspace_id, false, Arc::new(serde_json::value::to_raw_value(&wrapped_error).unwrap()), + None, unrecoverable, &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).clone(), worker_dir, @@ -797,7 +816,7 @@ pub async fn handle_job_error( if let Err(err) = updated_flow { if let Some(parent_job_id) = job.parent_job { if let Ok(Some(parent_job)) = - get_queued_job(&parent_job_id, &job.workspace_id, &db).await + get_mini_completed_job(&parent_job_id, &job.workspace_id, db).await { let e = json!({"message": err.to_string(), "name": "InternalErr"}); append_logs( @@ -809,7 +828,7 @@ pub async fn handle_job_error( .await; let _ = add_completed_job_error( db, - &MiniPulledJob::from(&parent_job), + &parent_job, mem_peak, canceled_by.clone(), e, diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index fb087a9d24..cd25663889 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -420,6 +420,7 @@ Your Gemfile syntax will continue to work as-is." &mut None, // Some(&mut stdout), None, + None, ) .await?; @@ -432,7 +433,7 @@ Your Gemfile syntax will continue to work as-is." if let Some(db) = conn.as_sql() { sqlx::query!( - "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", + "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = EXCLUDED.lockfile", req_hash, lock.clone(), ).fetch_optional(db).await?; @@ -871,6 +872,7 @@ mount {{ false, &mut Some(occupancy_metrics), None, + None, ) .await?; Ok(()) @@ -889,9 +891,23 @@ fn wrap(inner_content: &str) -> Result { require 'json' a = JSON.parse(File.read("args.json")) -res = main(SPREAD) -File.open("result.json", "w") do |file| - file.write(JSON.generate(res)) + +begin + res = main(SPREAD) + File.open("result.json", "w") do |file| + file.write(JSON.generate(res)) + end + +rescue => e + error = { + name: e.class.name, + stack: e.full_message, + message: e.message + } + File.open("result.json", "w") do |file| + file.write(JSON.generate(error)) + end + raise end "# .replace("INNER_CONTENT", inner_content) diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index f47399c54a..6a05de459c 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -175,7 +175,8 @@ pub async fn generate_cargo_lockfile( std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), ); } - let gen_lockfile_process = start_child_process(gen_lockfile_cmd, CARGO_PATH.as_str(), false).await?; + let gen_lockfile_process = + start_child_process(gen_lockfile_cmd, CARGO_PATH.as_str(), false).await?; handle_child( job_id, conn, @@ -190,6 +191,7 @@ pub async fn generate_cargo_lockfile( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -298,6 +300,7 @@ async fn get_build_dir( false, &mut None, None, + None, ) .await } @@ -410,6 +413,7 @@ pub async fn build_rust_crate( false, &mut Some(occupancy_metrics), None, + None, ) .await?; append_logs(&job.id, &job.workspace_id, "\n\n", conn).await; @@ -599,6 +603,7 @@ pub async fn handle_rust_job( false, &mut Some(occupancy_metrics), None, + None, ) .await?; read_result(job_dir, None).await diff --git a/backend/windmill-worker/src/sanitized_sql_params.rs b/backend/windmill-worker/src/sanitized_sql_params.rs index 465b67658f..8560f14798 100644 --- a/backend/windmill-worker/src/sanitized_sql_params.rs +++ b/backend/windmill-worker/src/sanitized_sql_params.rs @@ -1,11 +1,16 @@ use anyhow::anyhow; -use std::collections::HashMap; +use regex::Regex; +use std::collections::{HashMap, HashSet}; use serde_json::Value; use windmill_common::error; use windmill_parser::Arg; use windmill_parser_sql::{SANITIZED_ENUM_STR, SANITIZED_RAW_STRING_STR}; +lazy_static::lazy_static! { + static ref RE_SQL_CONTEXTUAL_VAR: Regex = Regex::new(r"%%WM_[A-Z_]+%%").unwrap(); +} + /// Identifier must be a continuous ASCII alphanumeric word, not starting with /// a number, that can contain underscores fn sanitize_identifier(arg: &Arg, input: &str) -> Result<(), error::Error> { @@ -28,14 +33,39 @@ fn sanitize_identifier(arg: &Arg, input: &str) -> Result<(), error::Error> { } } +fn replace_contextual_variables( + code: &mut String, + contextual_variables: &HashMap, +) -> () { + let vars = RE_SQL_CONTEXTUAL_VAR + .find_iter(&code) + .map(|m| m.as_str().to_string()) + .collect::>(); + + for var_pattern in vars { + let var_name = var_pattern + .strip_prefix("%%") + .unwrap() + .strip_suffix("%%") + .unwrap(); + let var_value = contextual_variables.get(var_name); + if let Some(var_value) = var_value { + *code = code.replace(&var_pattern, var_value); + } + } +} + pub fn sanitize_and_interpolate_unsafe_sql_args( code: &str, args: &Vec, args_map: &HashMap, + contextual_variables: &HashMap, ) -> Result<(String, Vec), error::Error> { let mut ret = code.to_string(); let mut args_to_skip = vec![]; + replace_contextual_variables(&mut ret, contextual_variables); + for arg in args { if let Some(typ) = &arg.otyp { let pattern = format!("%%{}%%", arg.name); diff --git a/backend/windmill-worker/src/scoped_dependency_map.rs b/backend/windmill-worker/src/scoped_dependency_map.rs new file mode 100644 index 0000000000..09436bed3b --- /dev/null +++ b/backend/windmill-worker/src/scoped_dependency_map.rs @@ -0,0 +1,458 @@ +use serde::Serialize; +use tokio::sync::RwLock; +use windmill_common::{ + apps::traverse_app_inline_scripts, + cache, + error::{Error, Result}, + flows::{FlowModuleValue, FlowValue}, + scripts::ScriptLang, +}; + +use std::collections::HashSet; + +use crate::worker_lockfiles::{ + extract_relative_imports, is_generated_from_raw_requirements, + LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT, +}; + +// TODO: To be removed in future versions +lazy_static::lazy_static! { + pub static ref WMDEBUG_NO_DMAP_DISSOLVE: bool = std::env::var("WMDEBUG_NO_DMAP_DISSOLVE").is_ok(); +} + +#[derive(Serialize)] +pub struct DependencyMap { + pub workspace_id: Option, + pub importer_path: Option, + pub importer_kind: Option, + pub imported_path: Option, + pub importer_node_id: Option, +} + +#[derive(Debug)] +pub struct ScopedDependencyMap { + dmap: HashSet<(String, String)>, + w_id: String, + importer_path: String, + importer_kind: String, +} + +impl ScopedDependencyMap { + /// Calls DB, however is assumed to be called once per dependency job + /// AND is scoped to smaller subset of data + /// So it is not too expensive + pub(crate) async fn fetch_maybe_rearranged<'a>( + w_id: &str, + importer_path: &str, + importer_kind: &str, + parent_path: &Option, + executor: impl sqlx::Executor<'a, Database = sqlx::Postgres>, + ) -> Result { + if parent_path + .as_ref() + .is_some_and(|x| !x.is_empty() && x != importer_path) + { + tracing::info!( + workspace_id = %w_id, + "detected top level rename from: {} to: {importer_path} on object of kind: {importer_kind}. reflecting in dependency_map.", + parent_path.clone().unwrap_or_default(), + ); + + let dmap = sqlx::query_as::<_, (String, String)>( + " +UPDATE dependency_map + SET importer_path = $1 + WHERE importer_path = $2 + AND importer_kind = $3::text::IMPORTER_KIND + AND workspace_id = $4 +RETURNING importer_node_id, imported_path + ", + ) + .bind(importer_path) + .bind(parent_path.clone().unwrap()) + .bind(importer_kind) + .bind(w_id) + .fetch_all(executor) + .await?; + Ok(Self { + dmap: HashSet::from_iter(dmap.into_iter()), + w_id: w_id.to_owned(), + importer_path: importer_path.to_owned(), + importer_kind: importer_kind.to_owned(), + }) + } else { + Self::fetch(w_id, importer_path, importer_kind, executor).await + } + } + + /// Almost same as [[Self::fetch_maybe_rearranged]], however only reads values, thus a bit faster. + pub async fn fetch<'a>( + w_id: &str, + importer_path: &str, + importer_kind: &str, + executor: impl sqlx::Executor<'a, Database = sqlx::Postgres>, + ) -> Result { + let dmap = sqlx::query_as::<_, (String, String)>( + " +SELECT importer_node_id, imported_path + FROM dependency_map + WHERE workspace_id = $1 + AND importer_path = $2 + AND importer_kind = $3::text::IMPORTER_KIND", + ) + .bind(w_id) + .bind(importer_path) + .bind(importer_kind) + .fetch_all(executor) + .await?; + + Ok(Self { + dmap: HashSet::from_iter(dmap.into_iter()), + w_id: w_id.to_owned(), + importer_path: importer_path.to_owned(), + importer_kind: importer_kind.to_owned(), + }) + } + + /// Add missing entries to `dependency_map` + /// Remove matching entries + pub(crate) async fn patch<'c>( + &mut self, + relative_imports: Option>, + node_id: String, // Flow Step/Node ID + mut tx: sqlx::Transaction<'c, sqlx::Postgres>, + ) -> Result> { + self.patch_tx_ref(relative_imports, &node_id, &mut tx) + .await?; + Ok(tx) + } + + pub(crate) async fn patch_tx_ref<'c>( + &mut self, + relative_imports: Option>, + node_id: &str, // Flow Step/Node ID + tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, + ) -> Result<()> { + let Some(mut relative_imports) = relative_imports else { + tracing::info!("relative imports are not found for: importer - {}, importer_node_id - {}, importer_kind - {}", + &self.importer_path, + &node_id, + &self.importer_kind, + ); + return Ok(()); + }; + + // This does: + // 1. remove all relative imports from relative_imports that ARE tracked in dependency_map + // 2. remove corresponding trackers from dependency_map + // + // After this operation `relative_imports` variable has only untracked imports. + // We will handle those in the next expression. + // + // After all `reduce`'s called ScopedDependencyMap has only extra/orphan imports + // these are going to be clean up by calling [dissolve] + // NOTE: `retain` iterates over vec and remove the ones whose closures returned false. + relative_imports.retain(|imported_path| { + !self + .dmap + // As dmap is HashSet, removing is O(1) operation + // thus making entire process very efficient + // NOTE: `remove` returns true if item was removed and false if wasn't. + .remove(&(node_id.to_owned(), imported_path.to_owned())) + }); + + // As mentioned above, usually this will always be empty. + if !relative_imports.is_empty() { + tracing::info!("adding missing entries to dependency_map: importer_node_id - {}, importer_kind - {}, new_imported_paths - {:?}", + &node_id, + &self.importer_kind, + &relative_imports, + ); + } + + for import in relative_imports { + sqlx::query!( + "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) + VALUES ($1, $2, $3::text::IMPORTER_KIND, $4, $5) ON CONFLICT DO NOTHING", + &self.w_id, + &self.importer_path, + &self.importer_kind, + import, + node_id + ) + .execute(&mut **tx) + .await?; + + tracing::info!("added entry to dependency_map: {import:?}"); + } + Ok(()) + } + + /// clean orphan entries from `dependency_map` + pub(crate) async fn dissolve<'a>( + self, + mut tx: sqlx::Transaction<'a, sqlx::Postgres>, + ) -> sqlx::Transaction<'a, sqlx::Postgres> { + if *WMDEBUG_NO_DMAP_DISSOLVE { + tracing::warn!( + "WMDEBUG_NO_DMAP_DISSOLVE usually should not be used. Behavior might be unstable." + ); + return tx; + } + + tracing::info!("dissolving dependency_map: {:?}", &self); + + // We _could_ shove it into single query, but this query is rarely called AND let's keep it simple for redability. + for (importer_node_id, imported_path) in self.dmap.into_iter() { + tracing::info!("cleaning orphan entry from dependency_map: importer_kind - {}, imported_path - {}, importer_node_id - {}", + &self.importer_kind, + &imported_path, + &importer_node_id, + ); + + // Dissolve MUST succeed. Error in dissolve MUST not block the execution. + if let Err(err) = sqlx::query!( + " + DELETE FROM dependency_map + WHERE workspace_id = $1 + AND importer_path = $2 + AND importer_kind = $3::text::IMPORTER_KIND + AND importer_node_id = $4 + AND imported_path = $5 + ", + &self.w_id, + &self.importer_path, + &self.importer_kind, + &importer_node_id, + &imported_path, + ) + .execute(&mut *tx) + .await + { + tracing::error!( + "error while cleaning dependency_map for: importer_node_id - {}, imported_path - {}, importer_path - {}: {err}", + importer_node_id, + imported_path, + self.importer_path, + ); + } + } + tx + } + + /// Selectively clean dependency_map for object + /// If `importer_node_id` is None will clear all nodes. + pub async fn clear_map_for_item<'c>( + item_path: &str, + w_id: &str, + importer_kind: &str, + mut tx: sqlx::Transaction<'c, sqlx::Postgres>, + importer_node_id: &Option, + ) -> sqlx::Transaction<'c, sqlx::Postgres> { + tracing::warn!( + importer = item_path, + kind = importer_kind, + node_id = importer_node_id, + workspace_id = w_id, + "discovered orphan entry in `dependency_map`. It will be healed automatically, however please report this issue to Windmill Team. It is also advised to rebuild maps in workspace settings in troubleshooting.", + ); + + // MUST succeed. Error MUST not block the execution. + if let Err(err) = sqlx::query!( + "DELETE FROM dependency_map + WHERE importer_path = $1 AND importer_kind = $3::text::IMPORTER_KIND + AND workspace_id = $2 AND ($4::text IS NULL OR importer_node_id = $4::text)", + item_path, + w_id, + importer_kind, + importer_node_id.clone(), + ) + .execute(&mut *tx) + .await + { + tracing::error!( + workspace_id = w_id, + "error while clearing discovered orphan: {err}" + ); + } + tx + } + + /// Run if you want to rebuild maps on specific workspace. + /// Potentially takes much time + pub async fn rebuild_map(w_id: &str, db: &sqlx::Pool) -> Result { + async fn inner<'c>(w_id: &str, db: &sqlx::Pool) -> Result { + // Scripts + tracing::info!(workspace_id = w_id, "Rebuilding dependency map for scripts"); + for r in sqlx::query!( + "SELECT path, hash FROM script WHERE workspace_id = $1 AND archived = false AND deleted = false", + w_id + ) + .fetch_all(db) + .await? + { + let (sd, smd) = cache::script::fetch(&db.clone().into(), r.hash.into()).await?; + let mut dmap = ScopedDependencyMap::fetch(w_id, &r.path, "script", db).await?; + let mut tx = db.begin().await?; + + if (smd.language.is_some_and(|v| v == ScriptLang::Bun) + && sd + .lock + .as_ref() + .is_some_and(|v| v.contains("generatedFromPackageJson"))) + || (smd.language.is_some_and(|v| v == ScriptLang::Python3) + && sd.lock.as_ref().is_some_and(|v| { + v.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT) + })) + { + // if the lock file is generated from a package.json/requirements.txt, we need to clear the dependency map + // because we do not want to have dependencies be recomputed automatically. Empty relative imports passed + // to update_script_dependency_map will clear the dependency map. + } else { + tx = dmap + .patch( + extract_relative_imports(&sd.code, &r.path, &smd.language), + "".into(), + tx, + ) + .await?; + } + if !*WMDEBUG_NO_DMAP_DISSOLVE { + dmap.dissolve(tx).await.commit().await?; + } + tracing::info!(workspace_id = w_id, "Rebuilt for script {}", &r.path); + } + + // Fetch only top level versions and paths + // It is not fetching value + tracing::info!(workspace_id = w_id, "Rebuilding dependency map for flows"); + for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM flow WHERE workspace_id = $1 AND archived = false", w_id).fetch_all(db).await? { + if let Some(version) = r.version { + // To reduce stress on db try to fetch from cache + // Since our flow versions are immutable it is safe to assume if we have cache for specific version/id it is up to date. + let flow_data = cache::flow::fetch_version(&db.clone().into(), version).await?; + + // Create map for specific flow + let mut dmap = ScopedDependencyMap::fetch(w_id, &r.path, "flow", db).await?; + + // Traverse retrieved flow modules + let mut tx = db.begin().await?; + let mut to_process = vec![]; + let mut modules_to_check = flow_data.flow.modules.iter().collect::>(); + if let Some(failure_module) = flow_data.flow.failure_module.as_ref() { + modules_to_check.push(failure_module.as_ref()); + } + if let Some(preprocessor_module) = flow_data.flow.preprocessor_module.as_ref() { + modules_to_check.push(preprocessor_module.as_ref()); + } + + FlowValue::traverse_leafs(modules_to_check, &mut |fmv, id| { + match fmv { + // Since we fetched from flow_version it is safe to assume all inline scripts are in form of RawScript. + FlowModuleValue::RawScript { content, language, lock ,.. } => { + if !is_generated_from_raw_requirements(Some(*language), lock) { + to_process.push(( + extract_relative_imports( + content, + &(r.path.clone() + "/flow"), + &Some(language.clone()), + ), + id.clone(), + )); + } + } + // But just in case we will also handle other cases. + FlowModuleValue::FlowScript { .. } => { + // Abort will cancel transaction. + return Err(Error::internal_err("FlowScript is not supposed to be in flow.")); + } + _ => {} + } + Ok(()) + })?; + + for (ri, id) in to_process { + tx = dmap.patch(ri, id, tx).await?; + } + + if !*WMDEBUG_NO_DMAP_DISSOLVE { + dmap.dissolve(tx).await.commit().await?; + } + + tracing::info!(workspace_id = w_id, "Rebuilt for flow {}", &r.path); + } else { + tracing::error!(workspace_id = w_id, "version is never supposed to be none. skipping flow."); + return Err(Error::internal_err("version was none")); + } + } + + // Apps + tracing::info!(workspace_id = w_id, "Rebuilding dependency map for apps"); + for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM app WHERE workspace_id = $1", w_id).fetch_all(db).await? { + if let Some(version) = r.version { + // TODO: Use cache when implemented. + let value = sqlx::query_scalar!( + "SELECT value FROM app_version WHERE id = $1 LIMIT 1", + version + ) + .fetch_one(db) + .await?; + + let mut dmap = ScopedDependencyMap::fetch(w_id, &r.path, "app", db).await?; + let mut tx = db.begin().await?; + let mut to_process = vec![]; + traverse_app_inline_scripts(&value, None, &mut |ais, id| { + to_process.push(( + extract_relative_imports( + &ais.content, + &(r.path.clone() + "/app"), + &ais.language, + ), + id, + )); + + Ok(()) + })?; + for (ri, id) in to_process { + tx = dmap.patch(ri, id.unwrap_or_default(), tx).await?; + } + if !*WMDEBUG_NO_DMAP_DISSOLVE { + dmap.dissolve(tx).await.commit().await?; + } + tracing::info!(workspace_id = w_id, "Rebuilt for app {}", &r.path); + } else { + tracing::error!( + workspace_id = w_id, + "version is never supposed to be none. skipping app." + ); + return Err(Error::internal_err("version was none")); + } + } + + Ok("Success".into()) + } + + lazy_static::lazy_static! { + pub static ref LOCKED: RwLock = RwLock::new(false); + } + + if *LOCKED.read().await { + tracing::warn!( + workspace_id = w_id, + "Tried to rebuild dependency map. However rebuild is already in progress." + ); + Ok("There is already one task pending, try again later.".into()) + } else { + tracing::info!(workspace_id = w_id, "Rebuilding dependency map"); + + if *WMDEBUG_NO_DMAP_DISSOLVE { + tracing::warn!("WMDEBUG_NO_DMAP_DISSOLVE usually should not be used. Behavior might be unstable. Please contact Windmill Team for support.") + } + + *LOCKED.write().await = true; + let r = inner(w_id, db).await; + *LOCKED.write().await = false; + r + } + } +} diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 0ba01b432a..e657ef7bb0 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -20,7 +20,7 @@ use windmill_queue::{CanceledBy, MiniPulledJob, HTTP_CLIENT}; use serde::{Deserialize, Serialize}; -use crate::common::build_args_values; +use crate::common::{build_args_values, get_reserved_variables}; use crate::common::{ build_http_client, resolve_job_timeout, s3_mode_args_to_worker_data, OccupancyMetrics, S3ModeWorkerData, @@ -132,12 +132,14 @@ fn do_snowflake_inner<'a>( skip_collect: bool, http_client: &'a Client, s3: Option, + reserved_variables: &HashMap, ) -> windmill_common::error::Result>>> { let sig = parse_snowflake_sig(&query) .map_err(|x| Error::ExecutionErr(x.to_string()))? .args; - let (query, args_to_skip) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args)?; + let (query, args_to_skip) = + &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args, reserved_variables)?; body.insert("statement".to_string(), json!(query)); @@ -282,6 +284,7 @@ pub async fn do_snowflake( worker_name: &str, column_order: &mut Option>, occupancy_metrics: &mut OccupancyMetrics, + parent_runnable_path: Option, ) -> windmill_common::error::Result> { let snowflake_args = build_args_values(job, client, conn).await?; @@ -403,6 +406,9 @@ pub async fn do_snowflake( let http_client = build_http_client(timeout_duration)?; + let reserved_variables = + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; + let result_f = if queries.len() > 1 { let futures = queries .iter() @@ -419,6 +425,7 @@ pub async fn do_snowflake( annotations.return_last_result && i < queries.len() - 1, &http_client, s3.clone(), + &reserved_variables, ) }) .collect::>>()?; @@ -449,6 +456,7 @@ pub async fn do_snowflake( false, &http_client, s3.clone(), + &reserved_variables, )? }; let r = run_future_with_polling_update_job_poller( diff --git a/backend/windmill-worker/src/universal_pkg_installer.rs b/backend/windmill-worker/src/universal_pkg_installer.rs index dc4b5dab2a..d4e33b4a72 100644 --- a/backend/windmill-worker/src/universal_pkg_installer.rs +++ b/backend/windmill-worker/src/universal_pkg_installer.rs @@ -159,6 +159,7 @@ pub async fn par_install_language_dependencies_all_at_once< false, &mut None, pipe_stdout, + None, ) .await { @@ -543,7 +544,7 @@ async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a + if let Err(e) = s3_pull_future.await { tracing::info!( workspace_id = %w_id, - "No tarball was found for {:?} on S3 or different problem occured {job_id}:\n{e}", + "No tarball was found for {:?} on S3 or different problem occurred {job_id}:\n{e}", &dep._s3_handle.clone() ); } else { @@ -587,6 +588,7 @@ async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a + false, &mut None, None, + None, ) .await { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 6532c1bb22..db568652e5 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -11,8 +11,11 @@ use anyhow::anyhow; use futures::TryFutureExt; +use tokio::time::sleep; use tokio::time::timeout; use windmill_common::client::AuthedClient; +use windmill_common::scripts::hash_to_codebase_id; +use windmill_common::scripts::is_special_codebase_hash; use windmill_common::utils::report_critical_error; use windmill_common::utils::retrieve_common_worker_prefix; use windmill_common::{ @@ -20,7 +23,6 @@ use windmill_common::{ apps::AppScriptId, cache::{future::FutureCachedExt, ScriptData, ScriptMetadata}, schema::{should_validate_schema, SchemaValidator}, - scripts::PREVIEW_IS_TAR_CODEBASE_HASH, utils::{create_directory_async, WarnAfterExt}, worker::{ make_pull_query, write_file, Connection, HttpClient, MAX_TIMEOUT, @@ -56,6 +58,9 @@ use std::{ time::Duration, }; use windmill_parser::MainArgSignature; +use windmill_queue::preprocess_dependency_job; +use windmill_queue::MiniCompletedJob; +use windmill_queue::PulledJobResultToJobErr; use uuid::Uuid; @@ -64,7 +69,7 @@ use windmill_common::{ error::{self, to_anyhow, Error}, flows::FlowNodeId, jobs::JobKind, - scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang, PREVIEW_IS_CODEBASE_HASH}, + scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, utils::StripPath, worker::{CLOUD_HOSTED, NO_LOGS, WORKER_CONFIG, WORKER_GROUP}, DB, IS_READY, @@ -101,9 +106,10 @@ use tokio::{ use rand::Rng; use crate::ai_executor::handle_ai_agent_job; +use crate::common::StreamNotifier; use crate::{ agent_workers::{queue_init_job, queue_periodic_job}, - bash_executor::{handle_bash_job, handle_powershell_job}, + bash_executor::handle_bash_job, bun_executor::handle_bun_job, common::{ build_args_map, cached_result_path, error_to_value, get_cached_resource_value_if_valid, @@ -118,6 +124,7 @@ use crate::{ job_logger::NO_LOGS_AT_ALL, js_eval::{eval_fetch_timeout, transpile_ts}, pg_executor::do_postgresql, + pwsh_executor::handle_powershell_job, result_processor::{process_result, start_background_processor}, schema::schema_validator_from_main_arg_sig, worker_flow::handle_flow, @@ -157,7 +164,7 @@ use crate::mysql_executor::do_mysql; #[cfg(feature = "duckdb")] use crate::duckdb_executor::do_duckdb; -#[cfg(feature = "oracledb")] +#[cfg(all(feature = "enterprise", feature = "oracledb"))] use crate::oracledb_executor::do_oracledb; #[cfg(feature = "enterprise")] @@ -261,6 +268,12 @@ const DOTNET_DEFAULT_PATH: &str = "/usr/bin/dotnet"; pub const SAME_WORKER_REQUIREMENTS: &'static str = "SameWorkerSender is required because this job may be part of a flow"; +#[derive(Deserialize, Clone)] +pub struct PowershellRepo { + pub url: String, + pub pat: String, +} + lazy_static::lazy_static! { pub static ref SLEEP_QUEUE: u64 = std::env::var("SLEEP_QUEUE") @@ -314,6 +327,12 @@ lazy_static::lazy_static! { } proxy_env }; + pub static ref WHITELIST_ENVS: HashMap = { + windmill_common::worker::load_env_vars( + windmill_common::worker::load_whitelist_env_vars_from_env(), + &HashMap::new(), + ) + }; pub static ref DENO_PATH: String = std::env::var("DENO_PATH").unwrap_or_else(|_| "/usr/bin/deno".to_string()); pub static ref BUN_PATH: String = std::env::var("BUN_PATH").unwrap_or_else(|_| "/usr/bin/bun".to_string()); pub static ref NPM_PATH: String = std::env::var("NPM_PATH").unwrap_or_else(|_| "/usr/bin/npm".to_string()); @@ -342,6 +361,8 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); pub static ref NUGET_CONFIG: Arc>> = Arc::new(RwLock::new(None)); + pub static ref POWERSHELL_REPO_URL: Arc>> = Arc::new(RwLock::new(None)); + pub static ref POWERSHELL_REPO_PAT: Arc>> = Arc::new(RwLock::new(None)); pub static ref MAVEN_REPOS: Arc>> = Arc::new(RwLock::new(None)); pub static ref NO_DEFAULT_MAVEN: AtomicBool = AtomicBool::new(std::env::var("NO_DEFAULT_MAVEN") .ok() @@ -452,6 +473,12 @@ pub enum JobCompletedSender { NeverUsed, } +impl JobCompletedSender { + pub fn is_sql(&self) -> bool { + matches!(self, Self::Sql(_)) + } +} + #[derive(Clone)] pub struct SqlJobCompletedSender { sender: flume::Sender, @@ -721,7 +748,7 @@ fn create_span(arc_job: &Arc, worker_name: &str, hostname: &str) pub async fn handle_all_job_kind_error( conn: &Connection, authed_client: &AuthedClient, - job: Arc, + job: MiniCompletedJob, err: Error, same_worker_tx: Option<&SameWorkerSender>, worker_dir: &str, @@ -734,7 +761,7 @@ pub async fn handle_all_job_kind_error( handle_job_error( db, authed_client, - job.as_ref(), + &job, 0, None, err, @@ -753,7 +780,7 @@ pub async fn handle_all_job_kind_error( .send_job( JobCompleted { preprocessed_args: None, - job: job.clone(), + job: job, result: Arc::new(windmill_common::worker::to_raw_value(&error_to_value( &err, ))), @@ -764,6 +791,8 @@ pub async fn handle_all_job_kind_error( cached_res_path: None, token: authed_client.token.clone(), duration: None, + has_stream: Some(false), + from_cache: None, }, false, ) @@ -790,6 +819,7 @@ pub fn start_interactive_worker_shell( loop { if let Ok(_) = killpill_rx.try_recv() { + tracing::info!("Received killpill, exiting worker shell"); break; } else { let pulled_job = match &conn { @@ -809,7 +839,18 @@ pub fn start_interactive_worker_shell( ) .await; - job.map(|x| x.job.map(NextJob::Sql)) + match job { + Ok(j) => match j.to_pulled_job() { + Ok(j) => Ok(j.map(NextJob::Sql)), + Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc)) => { + if let Err(err) = job_completed_tx.send_job(jc, true).await { + tracing::error!("An error occurred while sending job completed (missing concurrency key): {:#?}", err) + } + Ok(None) + } + }, + Err(err) => Err(err), + } } Connection::Http(client) => { crate::agent_workers::pull_job(&client, None, Some(true)) @@ -1477,7 +1518,6 @@ pub async fn run_worker( match &conn { Connection::Sql(db) => { let job = get_same_worker_job(db, &same_worker_job).await; - // tracing::error!("r: {:?}", r); if job.is_err() && !same_worker_job.recoverable { tracing::error!( worker = %worker_name, hostname = %hostname, @@ -1511,11 +1551,13 @@ pub async fn run_worker( } { if !killed_but_draining_same_worker_jobs { killed_but_draining_same_worker_jobs = true; - tracing::info!(worker = %worker_name, hostname = %hostname, "killpill received in worker main loop, sending killpill job"); - job_completed_tx - .kill() - .await - .expect("send kill to job completed tx"); + if job_completed_tx.is_sql() { + tracing::info!(worker = %worker_name, hostname = %hostname, "killpill received in worker main loop, sending killpill job"); + job_completed_tx + .kill() + .await + .expect("send kill to job completed tx"); + } } continue; } else if killed_but_draining_same_worker_jobs { @@ -1533,6 +1575,7 @@ pub async fn run_worker( Connection::Sql(db) => { let pull_time = Instant::now(); let likelihood_of_suspend = last_30jobs_suspended as f64 / 30.0; + let suspend_first = suspend_first_success || rand::random::() < likelihood_of_suspend || last_suspend_first.elapsed().as_secs_f64() > 5.0; @@ -1540,8 +1583,7 @@ pub async fn run_worker( if suspend_first { last_suspend_first = Instant::now(); } - - let job = match timeout( + let mut job = match timeout( Duration::from_secs(10), pull( &db, @@ -1563,6 +1605,30 @@ pub async fn run_worker( } }; + // Essential debouncing job preprocessing. + if let Ok(windmill_queue::PulledJobResult { + job: Some(ref mut pulled_job), + .. + }) = &mut job + { + match timeout( + core::time::Duration::from_secs(10), + preprocess_dependency_job(pulled_job, &db), + ) + .warn_after_seconds(2) + .await + { + Ok(Err(e)) => { + tracing::error!(worker = %worker_name, hostname = %hostname, "critical: debouncing job preprocessor failed: {e:?}"); + job = Err(e.into()); + } + Err(e) => { + tracing::error!(worker = %worker_name, hostname = %hostname, "critical: debouncing job preprocessor has timed out: {e:?}"); + job = Err(e.into()); + } + _ => {} + } + } add_time!(bench, "job pulled from DB"); let duration_pull_s = pull_time.elapsed().as_secs_f64(); let err_pull = job.is_ok(); @@ -1619,8 +1685,20 @@ pub async fn run_worker( } } } - job.map(|x| x.job.map(NextJob::Sql)) + match job { + Ok(pulled_job_result) => match pulled_job_result.to_pulled_job() { + Ok(j) => Ok(j.map(NextJob::Sql)), + Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc)) => { + if let Err(err) = job_completed_tx.send_job(jc, true).await { + tracing::error!("An error occurred while sending job completed (missing concurrency key): {:#?}", err) + } + Ok(None) + } + }, + Err(err) => Err(err), + } } + Connection::Http(client) => crate::agent_workers::pull_job(&client, None, None) .await .map_err(|e| error::Error::InternalErr(e.to_string())) @@ -1676,7 +1754,7 @@ pub async fn run_worker( .send_job( JobCompleted { preprocessed_args: None, - job: Arc::new(job.job()), + job: MiniCompletedJob::from(job.job()), success: true, result: Arc::new(empty_result()), result_columns: None, @@ -1685,6 +1763,8 @@ pub async fn run_worker( token: "".to_string(), canceled_by: None, duration: None, + has_stream: Some(false), + from_cache: None, }, true, ) @@ -1900,7 +1980,7 @@ pub async fn run_worker( handle_all_job_kind_error( &conn, &authed_client, - arc_job.clone(), + MiniCompletedJob::from(arc_job), err, Some(&same_worker_tx), &worker_dir, @@ -2040,8 +2120,14 @@ pub async fn run_worker( } tracing::info!(worker = %worker_name, hostname = %hostname, "waiting for interactive_shell to finish"); if let Some(interactive_shell) = interactive_shell { - if let Err(e) = interactive_shell.await { - tracing::error!("error in awaiting interactive_shell process: {e:?}") + match tokio::time::timeout(Duration::from_secs(10), interactive_shell).await { + Ok(Ok(_)) => {} + Ok(Err(e)) => { + tracing::error!("error in interactive_shell process: {e:?}") + } + Err(_) => { + tracing::error!("timed out awaiting interactive_shell process") + } } } tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exited", worker_name); @@ -2221,6 +2307,7 @@ async fn do_nativets( canceled_by: &mut Option, worker_name: &str, occupancy_metrics: &mut OccupancyMetrics, + has_stream: &mut bool, ) -> windmill_common::error::Result> { let args = build_args_map(job, client, conn).await?.map(Json); let job_args = if args.is_some() { @@ -2229,6 +2316,8 @@ async fn do_nativets( job.args.as_ref() }; + let stream_notifier = StreamNotifier::new(conn, job); + Ok(eval_fetch_timeout( env_code, code.clone(), @@ -2244,6 +2333,8 @@ async fn do_nativets( &job.workspace_id, true, occupancy_metrics, + stream_notifier, + has_stream, ) .await?) } @@ -2292,10 +2383,10 @@ pub async fn handle_queued_job( Connection::Sql(db) => { if job.parent_job.is_none() && job.created_by.starts_with("email-") { let daily_count = sqlx::query!( - "SELECT value FROM metrics WHERE id = 'email_trigger_usage' AND created_at > NOW() - INTERVAL '1 day' ORDER BY created_at DESC LIMIT 1" - ).fetch_optional(db) - .warn_after_seconds(5) - .await?.map(|x| serde_json::from_value::(x.value).unwrap_or(1)); + "SELECT value FROM metrics WHERE id = 'email_trigger_usage' AND created_at > NOW() - INTERVAL '1 day' ORDER BY created_at DESC LIMIT 1" + ).fetch_optional(db) + .warn_after_seconds(5) + .await?.map(|x| serde_json::from_value::(x.value).unwrap_or(1)); if let Some(count) = daily_count { if count >= 100 { @@ -2304,17 +2395,17 @@ pub async fn handle_queued_job( ))); } else { sqlx::query!( - "UPDATE metrics SET value = $1 WHERE id = 'email_trigger_usage' AND created_at > NOW() - INTERVAL '1 day'", - serde_json::json!(count + 1) - ) - .execute(db) - .warn_after_seconds(5) - .await?; + "UPDATE metrics SET value = $1 WHERE id = 'email_trigger_usage' AND created_at > NOW() - INTERVAL '1 day'", + serde_json::json!(count + 1) + ) + .execute(db) + .warn_after_seconds(5) + .await?; } } else { sqlx::query!( - "INSERT INTO metrics (id, value) VALUES ('email_trigger_usage', to_jsonb(1))" - ) + "INSERT INTO metrics (id, value) VALUES ('email_trigger_usage', to_jsonb(1))" + ) .execute(db) .warn_after_seconds(5) .await?; @@ -2344,15 +2435,19 @@ pub async fn handle_queued_job( | JobKind::Dependencies | JobKind::FlowPreview | JobKind::Flow - | JobKind::FlowDependencies, + | JobKind::FlowDependencies + | JobKind::SingleStepFlow, x, - ) => match x.map(|x| x.0) { - None | Some(PREVIEW_IS_CODEBASE_HASH) | Some(PREVIEW_IS_TAR_CODEBASE_HASH) => Some( - cache::job::fetch_preview(conn, &job.id, raw_lock, raw_code, raw_flow.clone()) - .await?, - ), - _ => None, - }, + ) => { + if x.map(|x| x.0).is_none_or(|x| is_special_codebase_hash(x)) { + Some( + cache::job::fetch_preview(conn, &job.id, raw_lock, raw_code, raw_flow.clone()) + .await?, + ) + } else { + None + } + } _ => None, }; @@ -2388,7 +2483,7 @@ pub async fn handle_queued_job( .send_job( JobCompleted { preprocessed_args: None, - job, + job: MiniCompletedJob::from(job), result, result_columns: None, mem_peak: 0, @@ -2397,6 +2492,8 @@ pub async fn handle_queued_job( cached_res_path: None, token: client.token.clone(), duration: None, + has_stream: Some(false), + from_cache: Some(true), }, true, ) @@ -2422,7 +2519,7 @@ pub async fn handle_queued_job( // Not a preview: fetch from the cache or the database. _ => cache::job::fetch_flow(db, &job.kind, job.runnable_id).await?, }; - handle_flow( + Box::pin(handle_flow( job, &flow_data, db, @@ -2432,7 +2529,7 @@ pub async fn handle_queued_job( worker_dir, job_completed_tx.clone(), worker_name, - ) + )) .warn_after_seconds(10) .await?; Ok(true) @@ -2471,6 +2568,23 @@ pub async fn handle_queued_job( logs.push_str("---\n"); } + // Only used for testing in tests/relative_imports.rs + // Give us some space to work with. + #[cfg(debug_assertions)] + if let Some(dbg_djob_sleep) = job + .args + .as_ref() + .map(|x| { + x.get("dbg_djob_sleep") + .map(|v| serde_json::from_str::(v.get()).ok()) + .flatten() + }) + .flatten() + { + tracing::debug!("Debug: {} going to sleep for {}", job.id, dbg_djob_sleep); + sleep(std::time::Duration::from_secs(dbg_djob_sleep as u64)).await; + } + tracing::debug!( workspace_id = %job.workspace_id, "handling job {}", @@ -2480,10 +2594,12 @@ pub async fn handle_queued_job( let mut column_order: Option> = None; let mut new_args: Option>> = None; + let mut has_stream = false; + // Box::pin all async branches to prevent large match enum on stack let result = match job.kind { JobKind::Dependencies => match conn { Connection::Sql(db) => { - handle_dependency_job( + Box::pin(handle_dependency_job( &job, preview_data.as_ref(), &mut mem_peak, @@ -2495,7 +2611,7 @@ pub async fn handle_queued_job( base_internal_url, &client.token, occupancy_metrics, - ) + )) .await } Connection::Http(_) => { @@ -2506,8 +2622,8 @@ pub async fn handle_queued_job( }, JobKind::FlowDependencies => match conn { Connection::Sql(db) => { - handle_flow_dependency_job( - &job, + Box::pin(handle_flow_dependency_job( + (*job).clone(), preview_data.as_ref(), &mut mem_peak, &mut canceled_by, @@ -2518,7 +2634,7 @@ pub async fn handle_queued_job( base_internal_url, &client.token, occupancy_metrics, - ) + )) .await } Connection::Http(_) => { @@ -2528,8 +2644,8 @@ pub async fn handle_queued_job( } }, JobKind::AppDependencies => match conn { - Connection::Sql(db) => handle_app_dependency_job( - &job, + Connection::Sql(db) => Box::pin(handle_app_dependency_job( + (*job).clone(), &mut mem_peak, &mut canceled_by, job_dir, @@ -2539,7 +2655,7 @@ pub async fn handle_queued_job( base_internal_url, &client.token, occupancy_metrics, - ) + )) .await .map(|()| serde_json::from_str("{}").unwrap()), Connection::Http(_) => { @@ -2557,7 +2673,7 @@ pub async fn handle_queued_job( .unwrap_or_else(|| serde_json::from_str("{}").unwrap())), JobKind::AIAgent => match conn { Connection::Sql(db) => { - handle_ai_agent_job( + Box::pin(handle_ai_agent_job( conn, db, job.as_ref(), @@ -2571,7 +2687,8 @@ pub async fn handle_queued_job( worker_name, hostname, killpill_rx, - ) + &mut has_stream, + )) .await } Connection::Http(_) => { @@ -2586,7 +2703,9 @@ pub async fn handle_queued_job( RawData::Script(data) => Some(data), _ => None, }); - let r = handle_code_execution_job( + + // Box::pin to move large future to heap + let r = Box::pin(handle_code_execution_job( job.as_ref(), preview_data, conn, @@ -2603,16 +2722,20 @@ pub async fn handle_queued_job( occupancy_metrics, killpill_rx, precomputed_agent_info, - ) + &mut has_stream, + )) .await; + occupancy_metrics.total_duration_of_running_jobs += metric_timer.elapsed().as_secs_f32(); r } }; + let cjob = MiniCompletedJob::from(job.to_owned()); + drop(job); //it's a test job, no need to update the db - if job.as_ref().workspace_id == "" { + if cjob.workspace_id == "" { return Ok(true); } @@ -2623,7 +2746,7 @@ pub async fn handle_queued_job( return Ok(false); } process_result( - job, + cjob, result.map(|x| Arc::new(x)), job_dir, job_completed_tx, @@ -2635,6 +2758,7 @@ pub async fn handle_queued_job( new_args, conn, Some(started.elapsed().as_millis() as i64), + has_stream, ) .await } @@ -2755,7 +2879,7 @@ async fn try_validate_schema( JobKind::Script_Hub => 3, JobKind::Preview => 4, JobKind::DeploymentCallback => 5, - JobKind::SingleScriptFlow => 6, + JobKind::SingleStepFlow => 6, JobKind::Dependencies => 7, JobKind::Flow => 8, JobKind::FlowPreview => 9, @@ -2822,31 +2946,32 @@ async fn handle_code_execution_job( occupancy_metrics: &mut OccupancyMetrics, killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, precomputed_agent_info: Option, + has_stream: &mut bool, ) -> error::Result> { let script_hash = || { job.runnable_id .ok_or_else(|| Error::internal_err("expected script hash")) }; + let (arc_data, arc_metadata, data, metadata): ( Arc, Arc, ScriptData, ScriptMetadata, ); + + // Box::pin the script fetching match to prevent large enum on stack let ( ScriptData { code, lock }, ScriptMetadata { language, envs, codebase, schema_validator, schema }, ) = match job.kind { JobKind::Preview => { - let codebase = match job.runnable_id.map(|x| x.0) { - Some(PREVIEW_IS_CODEBASE_HASH) => Some(job.id.to_string()), - Some(PREVIEW_IS_TAR_CODEBASE_HASH) => Some(format!("{}.tar", job.id)), - _ => None, - }; - + let codebase = job + .runnable_id + .and_then(|x| hash_to_codebase_id(&job.id.to_string(), x.0)); if codebase.is_none() && job.runnable_id.is_some() { (arc_data, arc_metadata) = - cache::script::fetch(conn, job.runnable_id.unwrap()).await?; + Box::pin(cache::script::fetch(conn, job.runnable_id.unwrap())).await?; (arc_data.as_ref(), arc_metadata.as_ref()) } else { arc_data = @@ -2863,19 +2988,26 @@ async fn handle_code_execution_job( } JobKind::Script_Hub => { let ContentReqLangEnvs { content, lockfile, language, envs, codebase, schema } = - get_hub_script_content_and_requirements(job.runnable_path.as_ref(), conn.as_sql()) - .await?; + Box::pin(get_hub_script_content_and_requirements( + job.runnable_path.as_ref(), + conn.as_sql(), + )) + .await?; data = ScriptData { code: content, lock: lockfile }; metadata = ScriptMetadata { language, envs, codebase, schema, schema_validator: None }; (&data, &metadata) } JobKind::Script => { - (arc_data, arc_metadata) = cache::script::fetch(conn, script_hash()?).await?; + (arc_data, arc_metadata) = Box::pin(cache::script::fetch(conn, script_hash()?)).await?; (arc_data.as_ref(), arc_metadata.as_ref()) } JobKind::FlowScript => { - arc_data = cache::flow::fetch_script(conn, FlowNodeId(script_hash()?.0)).await?; + arc_data = Box::pin(cache::flow::fetch_script( + conn, + FlowNodeId(script_hash()?.0), + )) + .await?; metadata = ScriptMetadata { language: job.script_lang, envs: None, @@ -2886,7 +3018,11 @@ async fn handle_code_execution_job( (arc_data.as_ref(), &metadata) } JobKind::AppScript => { - arc_data = cache::app::fetch_script(conn, AppScriptId(script_hash()?.0)).await?; + arc_data = Box::pin(cache::app::fetch_script( + conn, + AppScriptId(script_hash()?.0), + )) + .await?; metadata = ScriptMetadata { language: job.script_lang, envs: None, @@ -2904,8 +3040,11 @@ async fn handle_code_execution_job( .ok_or_else(|| Error::internal_err("expected script path".to_string()))?; if script_path.starts_with("hub/") { let ContentReqLangEnvs { content, lockfile, language, envs, codebase, schema } = - get_hub_script_content_and_requirements(Some(script_path), conn.as_sql()) - .await?; + Box::pin(get_hub_script_content_and_requirements( + Some(script_path), + conn.as_sql(), + )) + .await?; data = ScriptData { code: content, lock: lockfile }; metadata = ScriptMetadata { language, envs, codebase, schema, schema_validator: None }; @@ -2921,7 +3060,8 @@ async fn handle_code_execution_job( .await? .ok_or_else(|| Error::internal_err("expected script hash".to_string()))?; - (arc_data, arc_metadata) = cache::script::fetch(conn, ScriptHash(hash)).await?; + (arc_data, arc_metadata) = + Box::pin(cache::script::fetch(conn, ScriptHash(hash))).await?; (arc_data.as_ref(), arc_metadata.as_ref()) } } @@ -2958,6 +3098,7 @@ async fn handle_code_execution_job( worker_name, column_order, occupancy_metrics, + parent_runnable_path, ) .await; } else if language == Some(ScriptLang::Mysql) { @@ -2977,6 +3118,7 @@ async fn handle_code_execution_job( worker_name, column_order, occupancy_metrics, + parent_runnable_path, ) .await; } else if language == Some(ScriptLang::Bigquery) { @@ -3007,6 +3149,7 @@ async fn handle_code_execution_job( worker_name, column_order, occupancy_metrics, + parent_runnable_path, ) .await; } @@ -3030,6 +3173,7 @@ async fn handle_code_execution_job( worker_name, column_order, occupancy_metrics, + parent_runnable_path, ) .await; } @@ -3061,6 +3205,7 @@ async fn handle_code_execution_job( worker_name, occupancy_metrics, job_dir, + parent_runnable_path, ) .await; } @@ -3092,6 +3237,7 @@ async fn handle_code_execution_job( worker_name, column_order, occupancy_metrics, + parent_runnable_path, ) .await; } @@ -3116,6 +3262,7 @@ async fn handle_code_execution_job( worker_name, column_order, occupancy_metrics, + parent_runnable_path, ) .await; } @@ -3161,6 +3308,7 @@ async fn handle_code_execution_job( canceled_by, worker_name, occupancy_metrics, + has_stream, ) .await?; return Ok(result); @@ -3203,6 +3351,7 @@ mount {{ let envs = build_envs(envs.as_ref())?; + // Box::pin all language handlers to prevent large match enum on stack let result: error::Result> = match language { None => { return Err(Error::ExecutionErr( @@ -3216,7 +3365,7 @@ mount {{ )); #[cfg(feature = "python")] - handle_python_job( + Box::pin(handle_python_job( lock.as_ref(), job_dir, worker_dir, @@ -3234,11 +3383,12 @@ mount {{ new_args, occupancy_metrics, precomputed_agent_info, - ) + has_stream, + )) .await } Some(ScriptLang::Deno) => { - handle_deno_job( + Box::pin(handle_deno_job( lock.as_ref(), mem_peak, canceled_by, @@ -3253,11 +3403,12 @@ mount {{ envs, new_args, occupancy_metrics, - ) + has_stream, + )) .await } Some(ScriptLang::Bun) | Some(ScriptLang::Bunnative) => { - handle_bun_job( + Box::pin(handle_bun_job( lock.as_ref(), codebase.as_ref(), mem_peak, @@ -3275,11 +3426,12 @@ mount {{ new_args, occupancy_metrics, precomputed_agent_info, - ) + has_stream, + )) .await } Some(ScriptLang::Go) => { - handle_go_job( + Box::pin(handle_go_job( mem_peak, canceled_by, job, @@ -3294,11 +3446,11 @@ mount {{ worker_name, envs, occupancy_metrics, - ) + )) .await } Some(ScriptLang::Bash) => { - handle_bash_job( + Box::pin(handle_bash_job( mem_peak, canceled_by, job, @@ -3313,11 +3465,11 @@ mount {{ envs, occupancy_metrics, killpill_rx, - ) + )) .await } Some(ScriptLang::Powershell) => { - handle_powershell_job( + Box::pin(handle_powershell_job( mem_peak, canceled_by, job, @@ -3331,7 +3483,7 @@ mount {{ worker_name, envs, occupancy_metrics, - ) + )) .await } Some(ScriptLang::Php) => { @@ -3341,7 +3493,7 @@ mount {{ )); #[cfg(feature = "php")] - handle_php_job( + Box::pin(handle_php_job( lock.as_ref(), mem_peak, canceled_by, @@ -3356,7 +3508,7 @@ mount {{ envs, &shared_mount, occupancy_metrics, - ) + )) .await } Some(ScriptLang::Rust) => { @@ -3366,7 +3518,7 @@ mount {{ )); #[cfg(feature = "rust")] - handle_rust_job( + Box::pin(handle_rust_job( mem_peak, canceled_by, job, @@ -3381,7 +3533,7 @@ mount {{ worker_name, envs, occupancy_metrics, - ) + )) .await } Some(ScriptLang::Ansible) => { @@ -3391,7 +3543,7 @@ mount {{ )); #[cfg(feature = "python")] - handle_ansible_job( + Box::pin(handle_ansible_job( lock.as_ref(), job_dir, worker_dir, @@ -3407,11 +3559,11 @@ mount {{ base_internal_url, envs, occupancy_metrics, - ) + )) .await } Some(ScriptLang::CSharp) => { - handle_csharp_job( + Box::pin(handle_csharp_job( mem_peak, canceled_by, job, @@ -3426,7 +3578,7 @@ mount {{ worker_name, envs, occupancy_metrics, - ) + )) .await } Some(ScriptLang::Nu) => { @@ -3436,7 +3588,7 @@ mount {{ ); #[cfg(feature = "nu")] - handle_nu_job(JobHandlerInputNu { + Box::pin(handle_nu_job(JobHandlerInputNu { mem_peak, canceled_by, job, @@ -3451,7 +3603,7 @@ mount {{ worker_name, envs, occupancy_metrics, - }) + })) .await } Some(ScriptLang::Java) => { @@ -3462,7 +3614,7 @@ mount {{ .into()); #[cfg(feature = "java")] - handle_java_job(JobHandlerInputJava { + Box::pin(handle_java_job(JobHandlerInputJava { mem_peak, canceled_by, job, @@ -3477,7 +3629,7 @@ mount {{ worker_name, envs, occupancy_metrics, - }) + })) .await } Some(ScriptLang::Ruby) => { @@ -3488,7 +3640,7 @@ mount {{ .into()); #[cfg(feature = "ruby")] - handle_ruby_job(JobHandlerInputRuby { + Box::pin(handle_ruby_job(JobHandlerInputRuby { mem_peak, canceled_by, job, @@ -3503,7 +3655,7 @@ mount {{ worker_name, envs, occupancy_metrics, - }) + })) .await } // for related places search: ADD_NEW_LANG diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 70b72c3ecc..dd34f4dc24 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -36,8 +36,10 @@ use windmill_common::bench::BenchmarkIter; use windmill_common::cache::{self, RawData}; use windmill_common::client::AuthedClient; use windmill_common::db::Authed; +use windmill_common::flow_conversations::{add_message_to_conversation_tx, MessageType}; use windmill_common::flow_status::{ - ApprovalConditions, FlowStatusModuleWParent, Iterator as FlowIterator, JobResult, + ApprovalConditions, FlowJobDuration, FlowJobsDuration, FlowStatusModuleWParent, + Iterator as FlowIterator, JobResult, }; use windmill_common::flows::{add_virtual_items_if_necessary, Branch, FlowNodeId, StopAfterIf}; use windmill_common::jobs::{ @@ -63,7 +65,7 @@ use windmill_queue::schedule::get_schedule_opt; use windmill_queue::{ add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job, handle_maybe_scheduled_job, insert_concurrency_key, interpolate_args, CanceledBy, - MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError, + MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError, }; use windmill_audit::audit_oss::{audit_log, AuditAuthor}; @@ -79,6 +81,7 @@ pub async fn update_flow_status_after_job_completion( w_id: &str, success: bool, result: Arc>, + flow_job_duration: Option, unrecoverable: bool, same_worker_tx: &SameWorkerSender, worker_dir: &str, @@ -95,19 +98,21 @@ pub async fn update_flow_status_after_job_completion( job_id_for_status: job_id_for_status.clone(), success, result, + flow_job_duration, stop_early_override, has_triggered_error_handler: false, }; let mut unrecoverable = unrecoverable; loop { potentially_crash_for_testing(); - let nrec = match update_flow_status_after_job_completion_internal( + let nrec = match Box::pin(update_flow_status_after_job_completion_internal( db, client, rec.flow, &rec.job_id_for_status, w_id, rec.success, + rec.flow_job_duration.clone(), rec.result, unrecoverable, same_worker_tx, @@ -118,19 +123,20 @@ pub async fn update_flow_status_after_job_completion( job_completed_tx.clone(), #[cfg(feature = "benchmark")] bench, - ) + )) .await { Ok(j) => j, Err(e) => { tracing::error!("Error while updating flow status of {} after completion of {}, updating flow status again with error: {e:#}", rec.flow, &rec.job_id_for_status); - update_flow_status_after_job_completion_internal( + Box::pin(update_flow_status_after_job_completion_internal( db, client, rec.flow, &rec.job_id_for_status, w_id, false, + rec.flow_job_duration, Arc::new(to_raw_value(&Json(&WrappedError { error: json!(e.to_string()), }))), @@ -143,7 +149,7 @@ pub async fn update_flow_status_after_job_completion( job_completed_tx.clone(), #[cfg(feature = "benchmark")] bench, - ) + )) .await? } }; @@ -180,11 +186,13 @@ pub enum UpdateFlowStatusAfterJobCompletion { NonLastParallelBranch, PreprocessingStep, } + pub struct RecUpdateFlowStatusAfterJobCompletion { flow: uuid::Uuid, job_id_for_status: Uuid, success: bool, result: Arc>, + flow_job_duration: Option, stop_early_override: Option, has_triggered_error_handler: bool, } @@ -267,6 +275,7 @@ pub async fn update_flow_status_after_job_completion_internal( job_id_for_status: &Uuid, w_id: &str, mut success: bool, + mut flow_job_duration: Option, result: Arc>, unrecoverable: bool, same_worker_tx: &SameWorkerSender, @@ -279,6 +288,11 @@ pub async fn update_flow_status_after_job_completion_internal( ) -> error::Result { let mut has_triggered_error_handler = has_triggered_error_handler; add_time!(bench, "update flow status internal START"); + struct ChatAiInfo { + chat_input_enabled: bool, + conversation_id: Option, + is_ai_agent_step: bool, + } let ( should_continue_flow, flow_job, @@ -288,6 +302,7 @@ pub async fn update_flow_status_after_job_completion_internal( nresult, is_failure_step, _cleanup_module, + chat_ai_info, ) = { // tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id} {depth}"); @@ -328,7 +343,7 @@ pub async fn update_flow_status_after_job_completion_internal( let module_step = Step::from_i32_and_len(old_status.step, old_status.modules.len()); let current_module = match module_step { - Step::Step(i) => flow_value.modules.get(i), + Step::Step { idx: i, .. } => flow_value.modules.get(i), _ => None, }; @@ -342,7 +357,7 @@ pub async fn update_flow_status_after_job_completion_internal( .as_ref() .ok_or_else(|| Error::internal_err(format!("preprocessor module not found")))?, Step::FailureStep => &old_status.failure_module.module_status, - Step::Step(i) => old_status + Step::Step { idx: i, .. } => old_status .modules .get(i as usize) .ok_or_else(|| Error::internal_err(format!("module {i} not found")))?, @@ -364,7 +379,7 @@ pub async fn update_flow_status_after_job_completion_internal( .as_ref() .and_then(|x| x.skip_failures) .unwrap_or(false), - value.as_ref().and_then(|x| x.parallelism), + value.and_then(|x| x.parallelism), *parallel, ) } else { @@ -508,6 +523,7 @@ pub async fn update_flow_status_after_job_completion_internal( parallel, flow_jobs: Some(jobs), flow_jobs_success, + flow_jobs_duration, .. } if *parallel => { let (nindex, len) = match (iterator, branchall) { @@ -520,18 +536,22 @@ pub async fn update_flow_status_after_job_completion_internal( let nindex = if let Some(position) = position { sqlx::query_scalar!( - "UPDATE v2_job_status SET - flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), - ARRAY['modules', $1::TEXT, 'iterator', 'index'], - ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb - ) + " + UPDATE v2_job_status SET flow_status = + JSONB_SET(JSONB_SET(JSONB_SET(JSONB_SET( + flow_status, + ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), + ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb), + ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5), + ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6) WHERE id = $2 RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", old_status.step, flow, position as i32, - json!(success) + json!(success), + flow_job_duration.as_ref().map(|x| json!(x.started_at)).unwrap_or_default(), + flow_job_duration.as_ref().map(|x| json!(x.duration_ms)).unwrap_or_default(), ) } else { sqlx::query_scalar!( @@ -552,8 +572,20 @@ pub async fn update_flow_status_after_job_completion_internal( Error::internal_err(format!( "error while fetching iterator index: {e:#}" )) - })? - .ok_or_else(|| Error::internal_err(format!("requiring an index in InProgress")))?; + })?.ok_or_else(|| Error::internal_err(format!("requiring an index in InProgress for flow {flow} at step {}", old_status.step)))?; + + // let status_for_debug = sqlx::query!( + // "SELECT flow_status FROM v2_job_status WHERE id = $1", + // flow + // ) + // .fetch_one(&mut *tx) + // .await + // .map_err(|e| { + // Error::internal_err(format!("error while fetching flow status: {e:#}")) + // })?; + + // tracing::error!("status_for_debug: {:?}", status_for_debug.flow_status); + tracing::info!( "parallel iteration {job_id_for_status} of flow {flow} update nindex: {nindex} len: {len}", nindex = nindex, @@ -570,18 +602,33 @@ pub async fn update_flow_status_after_job_completion_internal( let nindex = if let Some(position) = position { sqlx::query_scalar!( - "UPDATE v2_job_status SET - flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), + "UPDATE v2_job_status SET flow_status = + CASE + WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN + JSONB_SET( + JSONB_SET(JSONB_SET(JSONB_SET( + flow_status, + ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), + ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5), + ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6), + ARRAY['modules', $1::TEXT, 'branchall', 'branch'], + ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb + ) + ELSE + JSONB_SET(JSONB_SET( + flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb ) + END WHERE id = $2 RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", old_status.step, flow, position as i32, - json!(success) + json!(success), + flow_job_duration.as_ref().map(|x| json!(x.started_at)).unwrap_or_default(), + flow_job_duration.as_ref().map(|x| json!(x.duration_ms)).unwrap_or_default(), ) } else { sqlx::query_scalar!( @@ -616,7 +663,22 @@ pub async fn update_flow_status_after_job_completion_internal( .execute(&mut *tx) .await?; if nindex == len { - let mut flow_jobs_success = flow_jobs_success.clone(); + let success_and_durations = match sqlx::query!( + "SELECT flow_status->'modules'->$2::int->'flow_jobs_success' as \"flow_jobs_success: Json>>\", flow_status->'modules'->$2::int->'flow_jobs_duration' as \"flow_jobs_duration: Json\" + FROM v2_job_status WHERE id = $1", + flow, + old_status.step + ) + .fetch_one(&mut *tx) + .await { + Err(e) => { + tracing::error!("error while fetching success and durations: {e:#}"); + (flow_jobs_success.clone(), flow_jobs_duration.clone()) + } + Ok(x) => (x.flow_jobs_success.map(|x| x.0), x.flow_jobs_duration.map(|x| x.0)), + }; + + let mut flow_jobs_success = success_and_durations.0; if let Some(flow_job_success) = flow_jobs_success.as_mut() { let position = jobs.iter().position(|x| x == job_id_for_status); if let Some(position) = position { @@ -625,6 +687,11 @@ pub async fn update_flow_status_after_job_completion_internal( } } } + let mut flow_jobs_duration = success_and_durations.1; + if let Some(flow_jobs_duration) = flow_jobs_duration.as_mut() { + let position = jobs.iter().position(|x| x == job_id_for_status); + flow_jobs_duration.set(position, &flow_job_duration); + } let branches = current_module .and_then(|x| x.get_branches_skip_failures().ok()) @@ -674,7 +741,7 @@ pub async fn update_flow_status_after_job_completion_internal( } let new_status = if - !(stop_early && stop_early_err_msg.is_some()) // if stop_early with error message, we want to set the job as failure and trigger the error handler if it exists + !(stop_early && stop_early_err_msg.is_some() && !skip_if_stop_early) // if stop_early with error and NOT skip_if_stopped, mark as failure && ( skip_loop_failures || sqlx::query_scalar!( @@ -685,7 +752,7 @@ pub async fn update_flow_status_after_job_completion_internal( .await .map_err(|e| { Error::internal_err(format!( - "error while fetching sucess from completed_jobs: {e:#}" + "error while fetching success from completed_jobs: {e:#}" )) })? .into_iter() @@ -698,10 +765,11 @@ pub async fn update_flow_status_after_job_completion_internal( job: job_id_for_status.clone(), flow_jobs: Some(jobs.clone()), flow_jobs_success: flow_jobs_success.clone(), + flow_jobs_duration: flow_jobs_duration.clone(), branch_chosen: None, approvers: vec![], failed_retries: vec![], - skipped: false, + skipped: stop_early && skip_if_stop_early, agent_actions: None, agent_actions_success: None, } @@ -712,6 +780,7 @@ pub async fn update_flow_status_after_job_completion_internal( job: job_id_for_status.clone(), flow_jobs: Some(jobs.clone()), flow_jobs_success: flow_jobs_success.clone(), + flow_jobs_duration: flow_jobs_duration.clone(), branch_chosen: None, failed_retries: vec![], agent_actions: None, @@ -797,13 +866,14 @@ pub async fn update_flow_status_after_job_completion_internal( && !stop_early => { if let Some(jobs) = flow_jobs { - set_success_in_flow_job_success( + set_success_and_duration_in_flow_job_success( flow_jobs_success, jobs, job_id_for_status, - &old_status, + old_status.step, flow, success, + flow_job_duration.clone(), &mut tx, ) .await?; @@ -821,13 +891,14 @@ pub async fn update_flow_status_after_job_completion_internal( && !stop_early => { if let Some(jobs) = flow_jobs { - set_success_in_flow_job_success( + set_success_and_duration_in_flow_job_success( flow_jobs_success, jobs, job_id_for_status, - &old_status, + old_status.step, flow, success, + flow_job_duration.clone(), &mut tx, ) .await?; @@ -872,6 +943,7 @@ pub async fn update_flow_status_after_job_completion_internal( let flow_jobs = module_status.flow_jobs(); let branch_chosen = module_status.branch_chosen(); let mut flow_jobs_success = module_status.flow_jobs_success(); + let mut flow_jobs_duration = module_status.flow_jobs_duration(); if let (Some(flow_job_success), Some(flow_jobs)) = (flow_jobs_success.as_mut(), flow_jobs.as_ref()) @@ -884,10 +956,18 @@ pub async fn update_flow_status_after_job_completion_internal( } } - // if stop_early with error message, we want to set the job as failure and trigger the error handler if it exists + if let (Some(flow_jobs_duration), Some(flow_jobs)) = + (flow_jobs_duration.as_mut(), flow_jobs.as_ref()) + { + let position = flow_jobs.iter().position(|x| x == job_id_for_status); + flow_jobs_duration.set(position, &flow_job_duration); + } + + // if stop_early with error message and NOT skip_if_stopped, mark as failure + // if skip_if_stopped=true, we want to mark as success (skipped), not failure if (success || (flow_jobs.is_some() && (skip_loop_failures || skip_seq_branch_failure))) - && !(stop_early && stop_early_err_msg.is_some()) + && !(stop_early && stop_early_err_msg.is_some() && !skip_if_stop_early) { let is_skipped = if current_module.as_ref().is_some_and(|m| m.skip_if.is_some()) { @@ -902,7 +982,7 @@ pub async fn update_flow_status_after_job_completion_internal( })? .unwrap_or(false) } else { - false + stop_early && skip_if_stop_early // Mark as skipped when stop_after_if with skip_if_stopped=true }; success = true; ( @@ -912,6 +992,7 @@ pub async fn update_flow_status_after_job_completion_internal( job: job_id_for_status.clone(), flow_jobs, flow_jobs_success, + flow_jobs_duration, branch_chosen, approvers: vec![], failed_retries: old_status.retry.failed_jobs.clone(), @@ -953,6 +1034,7 @@ pub async fn update_flow_status_after_job_completion_internal( job: job_id_for_status.clone(), flow_jobs, flow_jobs_success, + flow_jobs_duration, branch_chosen, failed_retries: old_status.retry.failed_jobs.clone(), agent_actions: module_status.agent_actions(), @@ -1168,14 +1250,14 @@ pub async fn update_flow_status_after_job_completion_internal( if let Some(t) = tag { tag = Some(interpolate_args(t, &args, &flow_job.workspace_id)); } - } else if let Some(ck) = concurrency_key { + } else if concurrent_limit.is_some() { let mut tx = db.begin().await?; insert_concurrency_key( &flow_job.workspace_id, &PushArgs::from(&HashMap::new()), &flow_job.runnable_path, JobKind::Flow, - Some(ck), + concurrency_key, &mut tx, flow, ) @@ -1284,6 +1366,11 @@ pub async fn update_flow_status_after_job_completion_internal( current_module_id = %current_module.map(|x| x.id.clone()).unwrap_or_default(), continue_on_error = %continue_on_error, should_continue_flow = %should_continue_flow, "computed if flow should continue"); + let chat_ai_info = ChatAiInfo { + chat_input_enabled: old_status.chat_input_enabled.unwrap_or(false), + conversation_id: old_status.memory_id, + is_ai_agent_step: current_module.is_some_and(|m| m.is_ai_agent()), + }; ( should_continue_flow, flow_job, @@ -1293,6 +1380,7 @@ pub async fn update_flow_status_after_job_completion_internal( nresult, is_failure_step, old_status.cleanup_module, + chat_ai_info, ) }; @@ -1362,51 +1450,78 @@ pub async fn update_flow_status_after_job_completion_internal( } if flow_job.is_canceled() { + let canceled_by = CanceledBy { + username: flow_job.canceled_by.clone(), + reason: flow_job.canceled_reason.clone(), + }; + let error = canceled_job_to_result(&flow_job); add_completed_job_error( db, - &flow_job, + &MiniCompletedJob::from(flow_job.clone()), 0, - Some(CanceledBy { - username: flow_job.canceled_by.clone(), - reason: flow_job.canceled_reason.clone(), - }), - canceled_job_to_result(&flow_job), + Some(canceled_by), + error, worker_name, true, None, ) .await?; } else { + let cflow_job: MiniCompletedJob = MiniCompletedJob::from(flow_job.clone()); + if flow_job.cache_ttl.is_some() && success { let flow = RawData::Flow(flow_data.clone()); let cached_res_path = cached_result_path(db, client, &flow_job, Some(&flow)).await; - save_in_cache(db, client, &flow_job, cached_res_path, nresult.clone()).await; + save_in_cache( + db, + client, + &MiniCompletedJob::from(cflow_job.clone()), + cached_res_path, + nresult.clone(), + ) + .await; } let success = success && (!is_failure_step || result_has_recover_true(nresult.clone())); add_time!(bench, "flow status update 1"); - if success { - add_completed_job( + + let skipped = stop_early && skip_if_stop_early; + add_tool_message_to_conversation( + db, + &job_id_for_status, + success, + skipped, + chat_ai_info.is_ai_agent_step, + &nresult, + chat_ai_info.chat_input_enabled, + chat_ai_info.conversation_id, + ) + .await?; + let duration = if success { + let (_, duration) = add_completed_job( db, - &flow_job, + &cflow_job, true, - stop_early && skip_if_stop_early, + skipped, Json(&nresult), None, 0, None, true, None, + false, + false, ) .await?; + duration } else { - add_completed_job( + let (_, duration) = add_completed_job( db, - &flow_job, + &cflow_job, false, - stop_early && skip_if_stop_early, + skipped, Json( &serde_json::from_str::(nresult.get()).unwrap_or_else( |e| json!({"error": format!("Impossible to serialize error: {e:#}")}), @@ -1417,14 +1532,20 @@ pub async fn update_flow_status_after_job_completion_internal( None, true, None, + false, + false, ) .await?; - } + duration + }; + flow_job_duration = flow_job + .started_at + .map(|x| FlowJobDuration { started_at: x, duration_ms: duration }); } true } else { tracing::debug!(id = %flow_job.id, "start handle flow"); - match handle_flow( + match Box::pin(handle_flow( flow_job.clone(), &flow_data, db, @@ -1434,7 +1555,7 @@ pub async fn update_flow_status_after_job_completion_internal( worker_dir, job_completed_tx, worker_name, - ) + )) .warn_after_seconds(10) .await { @@ -1447,8 +1568,17 @@ pub async fn update_flow_status_after_job_completion_internal( &db.into(), ) .await; - let _ = add_completed_job_error(db, &flow_job, 0, None, e, worker_name, true, None) - .await; + let _ = add_completed_job_error( + db, + &MiniCompletedJob::from(flow_job.clone()), + 0, + None, + e, + worker_name, + true, + None, + ) + .await; true } Ok(_) => false, @@ -1469,6 +1599,7 @@ pub async fn update_flow_status_after_job_completion_internal( flow: parent_job, job_id_for_status: flow, success: success && !is_failure_step, + flow_job_duration: flow_job_duration.clone(), result: nresult.clone(), stop_early_override: if stop_early { Some(skip_if_stop_early) @@ -1490,35 +1621,107 @@ fn find_flow_job_index(flow_jobs: &Vec, job_id_for_status: &Uuid) -> Optio flow_jobs.iter().position(|x| x == job_id_for_status) } -async fn set_success_in_flow_job_success<'c>( +async fn add_tool_message_to_conversation( + db: &DB, + job_id: &Uuid, + success: bool, + skipped: bool, + is_ai_agent_step: bool, + result: &Box, + chat_input_enabled: bool, + conversation_id: Option, +) -> error::Result<()> { + // Create assistant message if it's a flow and it's done, but only if last module is not an AI agent + if !skipped && chat_input_enabled { + // Get conversation_id from flow_status.memory_id + + if let Some(conversation_id) = conversation_id { + // Only create assistant message if last module is NOT an AI agent, or there was an error + if !is_ai_agent_step || success == false { + let value = serde_json::to_value(result.get()) + .map_err(|e| Error::internal_err(format!("Failed to serialize result: {e}")))?; + + let content = match value { + // If it's an Object with "output" key AND the output is a String, return it + serde_json::Value::Object(mut map) + if map.contains_key("output") + && matches!(map.get("output"), Some(serde_json::Value::String(_))) => + { + if let Some(serde_json::Value::String(s)) = map.remove("output") { + s + } else { + // prettify the whole result + serde_json::to_string_pretty(&map) + .unwrap_or_else(|e| format!("Failed to serialize result: {e}")) + } + } + // Otherwise, if the whole value is a String, return it + serde_json::Value::String(s) => s, + // Otherwise, prettify the whole result + v => serde_json::to_string_pretty(&v) + .unwrap_or_else(|e| format!("Failed to serialize result: {e}")), + }; + + // Insert new assistant message + let mut tx = db.begin().await?; + add_message_to_conversation_tx( + &mut tx, + conversation_id, + Some(job_id.clone()), + &content, + MessageType::Assistant, + None, + success, + ) + .await?; + tx.commit().await?; + } + } + } + + Ok(()) +} + +async fn set_success_and_duration_in_flow_job_success<'c>( flow_jobs_success: &Option>>, flow_jobs: &Vec, job_id_for_status: &Uuid, - old_status: &FlowStatus, + old_status_step: i32, flow: Uuid, success: bool, + flow_job_duration: Option, tx: &mut Transaction<'c, Postgres>, ) -> error::Result<()> { if flow_jobs_success.is_some() { let position = find_flow_job_index(flow_jobs, job_id_for_status); if let Some(position) = position { sqlx::query!( - "UPDATE v2_job_status SET - flow_status = JSONB_SET( + "UPDATE v2_job_status SET flow_status = + CASE WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN + JSONB_SET(JSONB_SET(JSONB_SET( flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4 - ) - WHERE id = $2", - old_status.step as i32, + ), + ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $5), + ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $6) + ELSE + JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4) + END + WHERE id = $2", + old_status_step as i32, flow, position as i32, - json!(success) + json!(success), + flow_job_duration.as_ref().map(|x| json!(x.duration_ms)).unwrap_or_default(), + flow_job_duration.as_ref().map(|x| json!(x.started_at)).unwrap_or_default() ) .execute(&mut **tx) .await .map_err(|e| { - Error::internal_err(format!("error while setting flow_jobs_success: {e:#}")) + Error::internal_err(format!( + "error while setting flow_jobs_success/timeline: {e:#}" + )) })?; } } @@ -1843,7 +2046,7 @@ pub async fn handle_flow( if let Some(schedule) = schedule { if let Err(err) = handle_maybe_scheduled_job( db, - &flow_job, + &MiniCompletedJob::from(flow_job.clone()), &schedule, flow_job.runnable_path.as_ref().unwrap(), &flow_job.workspace_id, @@ -1867,7 +2070,7 @@ pub async fn handle_flow( let mut rec = PushNextFlowJobRec { flow_job: flow_job, status: status }; loop { let PushNextFlowJobRec { flow_job, status } = rec; - let next = push_next_flow_job( + let next = Box::pin(push_next_flow_job( flow_job, status, flow, @@ -1877,7 +2080,7 @@ pub async fn handle_flow( same_worker_tx, worker_dir, worker_name, - ) + )) .warn_after_seconds(10) .await?; match next { @@ -1984,7 +2187,7 @@ async fn push_next_flow_job( tracing::info!(id = %flow_job.id, root_id = %job_root, step = ?step, "pushing next flow job"); let mut status_module = match step { - Step::Step(i) => status + Step::Step { idx: i, .. } => status .modules .get(i) .cloned() @@ -2007,8 +2210,10 @@ async fn push_next_flow_job( } }); - // if this is an empty module of if the module has already been completed, successfully, update the parent flow - if flow.modules.is_empty() || matches!(status_module, FlowStatusModule::Success { .. }) { + // if this is an empty module without preprocessor of if the module has already been completed, successfully, update the parent flow + if (flow.modules.is_empty() && !step.is_preprocessor_step()) + || matches!(status_module, FlowStatusModule::Success { .. }) + { return Ok(PushNextFlowJob::Done(Some(UpdateFlow { flow: flow_job.id, success: true, @@ -2033,7 +2238,7 @@ async fn push_next_flow_job( }))); } - if matches!(step, Step::Step(0)) { + if matches!(step, Step::Step { idx: 0, .. }) { if !flow_job.is_flow_step() && flow_job.schedule_path().is_some() { let schedule_path = flow_job.schedule_path(); let no_flow_overlap = sqlx::query_scalar!( @@ -2123,7 +2328,7 @@ async fn push_next_flow_job( let arc_last_job_result = if status_module.is_failure() { // if job is being retried, pass the result of its previous failure last_job_result.unwrap_or_else(|| Arc::new(to_raw_value(&json!("{}")))) - } else if matches!(step, Step::Step(0)) || step.is_preprocessor_step() { + } else if matches!(step, Step::Step { idx: 0, .. }) || step.is_preprocessor_step() { // if it's the first job executed in the flow, pass the flow args Arc::new(to_raw_value(&flow_job.args)) } else { @@ -2442,7 +2647,9 @@ async fn push_next_flow_job( let current_id = &module.id; let mut previous_id = match step { - Step::Step(i) if i >= 1 => flow.modules.get(i - 1).map(|m| m.id.clone()).unwrap(), + Step::Step { idx: i, .. } if i >= 1 => { + flow.modules.get(i - 1).map(|m| m.id.clone()).unwrap() + } _ => String::new(), }; @@ -2456,7 +2663,7 @@ async fn push_next_flow_job( ) { None } else { - let sleep_input_transform = if let Step::Step(i) = step { + let sleep_input_transform = if let Step::Step { idx: i, .. } = step { i.checked_sub(1) .and_then(|i| flow.modules.get(i)) .and_then(|m| m.sleep.clone()) @@ -2647,6 +2854,9 @@ async fn push_next_flow_job( to_raw_value(&"preprocessor"), ); Ok(Marc::new(hm)) + } else if module.pass_flow_input_directly.unwrap_or(false) { + // If pass_flow_input_directly is set, use flow args directly + Ok(arc_flow_job_args.clone()) } else { let value = module.get_value(); match &value { @@ -2749,6 +2959,11 @@ async fn push_next_flow_job( } else { Some(vec![]) }, + flow_jobs_duration: if branch_chosen.is_some() { + None + } else { + Some(FlowJobsDuration { started_at: vec![], duration_ms: vec![] }) + }, branch_chosen: branch_chosen, approvers: vec![], failed_retries: vec![], @@ -3051,6 +3266,8 @@ async fn push_next_flow_job( new_job_priority_override, job_perms.as_ref(), false, + None, + None, ) .warn_after_seconds(2) .await?; @@ -3068,18 +3285,35 @@ async fn push_next_flow_job( tracing::debug!(id = %flow_job.id, root_id = %job_root, "pushed next flow job: {uuid}"); - if value_with_parallel.type_ == "forloopflow" { - if let Some(p) = value_with_parallel.parallelism { - tracing::debug!(id = %flow_job.id, root_id = %job_root, "updating suspend for forloopflow job {uuid}"); + if value_with_parallel.type_ == "forloopflow" + && value_with_parallel.parallel.unwrap_or(false) + { + if let Some(parallelism_transform) = &value_with_parallel.parallelism { + tracing::debug!(id = %flow_job.id, root_id = %job_root, "evaluating parallelism expression for forloopflow job {uuid}"); - if i as u16 >= p { + let ctx = get_transform_context(&flow_job, &previous_id, &status) + .warn_after_seconds(3) + .await?; + + let evaluated_parallelism = evaluate_input_transform::( + parallelism_transform, + arc_last_job_result.clone(), + Some(arc_flow_job_args.clone()), + Some(client), + Some(&ctx), + ) + .await?; + + tracing::debug!(id = %flow_job.id, root_id = %job_root, "updating suspend for forloopflow job {uuid} with parallelism {evaluated_parallelism}"); + + if i as u16 >= evaluated_parallelism { sqlx::query!( "UPDATE v2_job_queue SET suspend = $1, suspend_until = now() + interval '14 day', running = true WHERE id = $2", - (i as u16 - p + 1) as i32, + (i as u16 - evaluated_parallelism + 1) as i32, uuid, ) .execute(&mut *inner_tx) @@ -3144,6 +3378,7 @@ async fn push_next_flow_job( mut flow_jobs, while_loop, mut flow_jobs_success, + mut flow_jobs_duration, .. }, .. @@ -3155,11 +3390,15 @@ async fn push_next_flow_job( if let Some(flow_jobs_success) = &mut flow_jobs_success { flow_jobs_success.push(None); } + if let Some(flow_jobs_duration) = &mut flow_jobs_duration { + flow_jobs_duration.push(&None); + } FlowStatusModule::InProgress { job: uuid, iterator: Some(FlowIterator { index, itered }), flow_jobs: Some(flow_jobs), flow_jobs_success, + flow_jobs_duration, branch_chosen: None, branchall: None, id: status_module.id(), @@ -3175,6 +3414,7 @@ async fn push_next_flow_job( iterator, flow_jobs_success: Some(vec![None; uuids.len()]), flow_jobs: Some(uuids.clone()), + flow_jobs_duration: Some(FlowJobsDuration::new(uuids.len())), branch_chosen: None, branchall, id: status_module.id(), @@ -3188,6 +3428,7 @@ async fn push_next_flow_job( mut flow_jobs, status, mut flow_jobs_success, + mut flow_jobs_duration, .. }) => { let uuid = one_uuid?; @@ -3195,11 +3436,15 @@ async fn push_next_flow_job( if let Some(flow_jobs_success) = &mut flow_jobs_success { flow_jobs_success.push(None); } + if let Some(flow_jobs_duration) = &mut flow_jobs_duration { + flow_jobs_duration.push(&None); + } FlowStatusModule::InProgress { job: uuid, iterator: None, flow_jobs: Some(flow_jobs), flow_jobs_success, + flow_jobs_duration, branch_chosen: None, branchall: Some(status), id: status_module.id(), @@ -3216,6 +3461,7 @@ async fn push_next_flow_job( iterator: None, flow_jobs: None, flow_jobs_success: None, + flow_jobs_duration: None, branch_chosen: Some(branch), branchall: None, id: status_module.id(), @@ -3270,7 +3516,7 @@ async fn push_next_flow_job( .warn_after_seconds(3) .await?; } - Step::Step(i) => { + Step::Step { idx: i, .. } => { sqlx::query!( "UPDATE v2_job_status SET flow_status = JSONB_SET( @@ -3392,6 +3638,7 @@ struct ForloopNextIteration { itered: Vec>, flow_jobs: Vec, flow_jobs_success: Option>>, + flow_jobs_duration: Option, new_args: Iter, while_loop: bool, } @@ -3407,6 +3654,7 @@ struct NextBranch { status: BranchAllStatus, flow_jobs: Vec, flow_jobs_success: Option>>, + flow_jobs_duration: Option, } #[derive(Debug)] @@ -3658,13 +3906,18 @@ async fn compute_next_flow_transform( FlowModuleValue::WhileloopFlow { modules, modules_node, .. } => { // if it's a simple single step flow, we will collapse it as an optimization and need to pass flow_input as an arg let is_simple = is_simple_modules(&modules, flow.failure_module.as_ref()); - let (flow_jobs, flow_jobs_success) = match status_module { + let (flow_jobs, flow_jobs_success, flow_jobs_duration) = match status_module { FlowStatusModule::InProgress { flow_jobs: Some(flow_jobs), flow_jobs_success, + flow_jobs_duration, .. - } => (flow_jobs.clone(), flow_jobs_success.clone()), - _ => (vec![], Some(vec![])), + } => ( + flow_jobs.clone(), + flow_jobs_success.clone(), + flow_jobs_duration.clone(), + ), + _ => (vec![], Some(vec![]), Some(FlowJobsDuration::new(0))), }; let next_loop_idx = flow_jobs.len(); next_loop_iteration( @@ -3675,6 +3928,7 @@ async fn compute_next_flow_transform( itered: vec![], flow_jobs: flow_jobs, flow_jobs_success: flow_jobs_success, + flow_jobs_duration: flow_jobs_duration, new_args: Iter { index: next_loop_idx as i32, value: windmill_common::worker::to_raw_value(&next_loop_idx), @@ -3873,72 +4127,78 @@ async fn compute_next_flow_transform( )) } FlowModuleValue::BranchAll { branches, parallel, .. } => { - let (branch_status, flow_jobs, flow_jobs_success) = match status_module { - FlowStatusModule::WaitingForPriorSteps { .. } - | FlowStatusModule::WaitingForEvents { .. } - | FlowStatusModule::WaitingForExecutor { .. } => { - if branches.is_empty() { - return Ok(NextFlowTransform::EmptyInnerFlows { branch_chosen: None }); - } else if parallel { - let len = branches.len(); - let payloads: Vec = branches - .into_iter() - .enumerate() - .filter_map(|(i, Branch { modules, modules_node, .. })| { - let Some(payload) = payload_from_modules( - modules, - modules_node, - flow.failure_module.as_ref(), - flow.same_worker, - || format!("{}-{i}", status.step), - || format!("{}/branchall-{}", flow_job.runnable_path(), i), - false, - ) else { - return None; - }; - Some(JobPayloadWithTag { - payload, - tag: None, - delete_after_use, - timeout: None, - on_behalf_of: None, - }) - }) - .collect::>(); - if payloads.is_empty() { + let (branch_status, flow_jobs, flow_jobs_success, flow_jobs_duration) = + match status_module { + FlowStatusModule::WaitingForPriorSteps { .. } + | FlowStatusModule::WaitingForEvents { .. } + | FlowStatusModule::WaitingForExecutor { .. } => { + if branches.is_empty() { return Ok(NextFlowTransform::EmptyInnerFlows { branch_chosen: None }); + } else if parallel { + let len = branches.len(); + let payloads: Vec = branches + .into_iter() + .enumerate() + .filter_map(|(i, Branch { modules, modules_node, .. })| { + let Some(payload) = payload_from_modules( + modules, + modules_node, + flow.failure_module.as_ref(), + flow.same_worker, + || format!("{}-{i}", status.step), + || format!("{}/branchall-{}", flow_job.runnable_path(), i), + false, + ) else { + return None; + }; + Some(JobPayloadWithTag { + payload, + tag: None, + delete_after_use, + timeout: None, + on_behalf_of: None, + }) + }) + .collect::>(); + if payloads.is_empty() { + return Ok(NextFlowTransform::EmptyInnerFlows { + branch_chosen: None, + }); + } + return Ok(NextFlowTransform::Continue( + ContinuePayload::ParallelJobs(payloads), + NextStatus::AllFlowJobs { + branchall: Some(BranchAllStatus { branch: 0, len }), + iterator: None, + simple_input_transforms: None, + }, + )); + } else { + ( + BranchAllStatus { branch: 0, len: branches.len() }, + vec![], + Some(vec![]), + Some(FlowJobsDuration::new(0)), + ) } - return Ok(NextFlowTransform::Continue( - ContinuePayload::ParallelJobs(payloads), - NextStatus::AllFlowJobs { - branchall: Some(BranchAllStatus { branch: 0, len }), - iterator: None, - simple_input_transforms: None, - }, - )); - } else { - ( - BranchAllStatus { branch: 0, len: branches.len() }, - vec![], - Some(vec![]), - ) } - } - FlowStatusModule::InProgress { - branchall: Some(BranchAllStatus { branch, len }), - flow_jobs: Some(flow_jobs), - flow_jobs_success, - .. - } if !parallel => ( - BranchAllStatus { branch: branch + 1, len: len.clone() }, - flow_jobs.clone(), - flow_jobs_success.clone(), - ), + FlowStatusModule::InProgress { + branchall: Some(BranchAllStatus { branch, len }), + flow_jobs: Some(flow_jobs), + flow_jobs_success, + flow_jobs_duration, + .. + } if !parallel => ( + BranchAllStatus { branch: branch + 1, len: len.clone() }, + flow_jobs.clone(), + flow_jobs_success.clone(), + flow_jobs_duration.clone(), + ), - _ => Err(Error::BadRequest(format!( - "Unrecognized module status for BranchAll {status_module:?}" - )))?, - }; + _ => Err(Error::BadRequest(format!( + "Unrecognized module status for BranchAll {status_module:?}" + )))?, + }; let Branch { modules, modules_node, .. } = branches .into_iter() @@ -3968,7 +4228,6 @@ async fn compute_next_flow_transform( branch_chosen: Some(BranchChosen::Default), }); }; - Ok(NextFlowTransform::Continue( ContinuePayload::SingleJob(JobPayloadWithTag { payload, @@ -3981,6 +4240,7 @@ async fn compute_next_flow_transform( status: branch_status, flow_jobs, flow_jobs_success, + flow_jobs_duration, }), )) } @@ -4126,6 +4386,7 @@ async fn next_forloop_status( itered, flow_jobs: vec![], flow_jobs_success: Some(vec![]), + flow_jobs_duration: Some(FlowJobsDuration::new(0)), new_args: iter, while_loop: false, }) @@ -4138,6 +4399,7 @@ async fn next_forloop_status( iterator: Some(FlowIterator { itered, index }), flow_jobs: Some(flow_jobs), flow_jobs_success, + flow_jobs_duration, .. } if !*parallel => { let itered_new = if itered.is_empty() { @@ -4188,6 +4450,7 @@ async fn next_forloop_status( itered: itered_new.clone(), flow_jobs: flow_jobs.clone(), flow_jobs_success: flow_jobs_success.clone(), + flow_jobs_duration: flow_jobs_duration.clone(), new_args: Iter { index: index as i32, value: next.to_owned() }, while_loop: false, }) @@ -4298,6 +4561,8 @@ pub fn raw_script_to_payload( concurrency_time_window_s, cache_ttl: module.cache_ttl.map(|x| x as i32), dedicated_worker: None, + custom_debounce_key: None, + debounce_delay_s: None, }), tag, delete_after_use, @@ -4362,6 +4627,8 @@ pub async fn script_to_payload( concurrency_key, concurrent_limit, concurrency_time_window_s, + debounce_key, + debounce_delay_s, cache_ttl, language, dedicated_worker, @@ -4379,15 +4646,17 @@ pub async fn script_to_payload( }; ( // We only apply the preprocessor if it's explicitly set to true in the module, - // which can only happen if the the flow is a SingleScriptFlow triggered by a trigger with retries or error handling. + // which can only happen if the the flow is a SingleStepFlow triggered by a trigger with retries or error handling. // In that case, apply_preprocessor is still only set to true if the script has a preprocesor. - // We only check for script hash because SingleScriptFlow triggers specifies the script hash + // We only check for script hash because SingleStepFlow triggers specifies the script hash JobPayload::ScriptHash { hash, path: script_path, custom_concurrency_key: concurrency_key, concurrent_limit, concurrency_time_window_s, + custom_debounce_key: debounce_key, + debounce_delay_s, cache_ttl: module.cache_ttl.map(|x| x as i32).ok_or(cache_ttl).ok(), language, dedicated_worker, @@ -4418,7 +4687,7 @@ pub async fn script_to_payload( }) } -async fn get_transform_context( +pub async fn get_transform_context( flow_job: &MiniPulledJob, previous_id: &str, status: &FlowStatus, @@ -4485,7 +4754,7 @@ fn needs_resume(flow: &FlowValue, status: &FlowStatus) -> Option<(Suspend, Uuid) } // returns the result of the previous step of a running flow (if the job was successful) -async fn get_previous_job_result( +pub async fn get_previous_job_result( db: &sqlx::Pool, w_id: &str, flow_status: &FlowStatus, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index be0cc38cc3..7b1acc19e8 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -5,20 +5,23 @@ use std::path::{Component, Path, PathBuf}; #[cfg(feature = "python")] use crate::ansible_executor::{get_git_repos_lock, AnsibleDependencyLocks}; +use crate::scoped_dependency_map::ScopedDependencyMap; use async_recursion::async_recursion; +use chrono::{Duration, Utc}; use itertools::Itertools; use serde_json::value::RawValue; use serde_json::{from_value, json, Value}; use sha2::Digest; use sqlx::types::Json; +use tokio::time::timeout; use uuid::Uuid; use windmill_common::assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind}; use windmill_common::error::Error; use windmill_common::error::Result; use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; -use windmill_common::get_latest_deployed_hash_for_path; use windmill_common::jobs::JobPayload; -use windmill_common::scripts::{hash_script, NewScript, ScriptHash}; +use windmill_common::scripts::ScriptHash; +use windmill_common::utils::WarnAfterExt; #[cfg(feature = "python")] use windmill_common::worker::PythonAnnotations; use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, Connection}; @@ -45,6 +48,9 @@ lazy_static::lazy_static! { static ref WMDEBUG_NO_NEW_FLOW_VERSION_ON_DJ: bool = std::env::var("WMDEBUG_NO_NEW_FLOW_VERSION_ON_DJ").is_ok(); static ref WMDEBUG_NO_NEW_APP_VERSION_ON_DJ: bool = std::env::var("WMDEBUG_NO_NEW_APP_VERSION_ON_DJ").is_ok(); static ref WMDEBUG_NO_COMPONENTS_TO_RELOCK: bool = std::env::var("WMDEBUG_NO_COMPONENTS_TO_RELOCK").is_ok(); + static ref DEPENDENCY_JOB_DEBOUNCE_DELAY: usize = std::env::var("DEPENDENCY_JOB_DEBOUNCE_DELAY").ok().and_then(|flag| flag.parse().ok()).unwrap_or( + if cfg!(test) { /* if test we want increased debouncing delay */ 15 } else { 5 } + ); } use crate::common::OccupancyMetrics; @@ -69,115 +75,6 @@ use crate::{ go_executor::install_go_dependencies, }; -pub async fn update_script_dependency_map( - job_id: &Uuid, - db: &DB, - w_id: &str, - parent_path: &Option, - script_path: &str, - relative_imports: Vec, -) -> error::Result<()> { - let importer_kind = "script"; - - let mut tx = db.begin().await?; - tx = clear_dependency_parent_path(parent_path, script_path, w_id, importer_kind, tx).await?; - - tx = clear_dependency_map_for_item(script_path, w_id, importer_kind, tx, &None).await?; - - if !relative_imports.is_empty() { - let mut logs = "".to_string(); - logs.push_str("\n--- RELATIVE IMPORTS ---\n\n"); - logs.push_str(&relative_imports.join("\n")); - - tx = add_relative_imports_to_dependency_map( - script_path, - w_id, - relative_imports, - importer_kind, - tx, - &mut logs, - None, - ) - .await?; - append_logs(job_id, w_id, logs, &db.into()).await; - } - tx.commit().await?; - - Ok(()) -} - -async fn add_relative_imports_to_dependency_map<'c>( - script_path: &str, - w_id: &str, - relative_imports: Vec, - importer_kind: &str, - mut tx: sqlx::Transaction<'c, sqlx::Postgres>, - logs: &mut String, - node_id: Option, -) -> error::Result> { - for import in relative_imports { - sqlx::query!( - "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) - VALUES ($1, $2, $4::text::IMPORTER_KIND, $3, $5) ON CONFLICT DO NOTHING", - w_id, - script_path, - import, - importer_kind, - node_id.clone().unwrap_or_default() - ) - .execute(&mut *tx) - .await?; - logs.push_str(&format!("{}\n", import)); - } - Ok(tx) -} - -async fn clear_dependency_map_for_item<'c>( - item_path: &str, - w_id: &str, - importer_kind: &str, - mut tx: sqlx::Transaction<'c, sqlx::Postgres>, - importer_node_id: &Option, -) -> Result> { - sqlx::query!( - "DELETE FROM dependency_map - WHERE importer_path = $1 AND importer_kind = $3::text::IMPORTER_KIND - AND workspace_id = $2 AND ($4::text IS NULL OR importer_node_id = $4::text)", - item_path, - w_id, - importer_kind, - importer_node_id.clone() - ) - .execute(&mut *tx) - .await?; - Ok(tx) -} - -async fn clear_dependency_parent_path<'c>( - parent_path: &Option, - item_path: &str, - w_id: &str, - importer_kind: &str, - mut tx: sqlx::Transaction<'c, sqlx::Postgres>, -) -> Result> { - if parent_path - .as_ref() - .is_some_and(|x| !x.is_empty() && x != item_path) - { - sqlx::query!( - "DELETE FROM dependency_map - WHERE importer_path = $1 AND importer_kind = $3::text::IMPORTER_KIND - AND workspace_id = $2", - parent_path.clone().unwrap(), - w_id, - importer_kind - ) - .execute(&mut *tx) - .await?; - } - Ok(tx) -} - fn try_normalize(path: &Path) -> Option { let mut ret = PathBuf::new(); @@ -237,6 +134,7 @@ pub fn extract_relative_imports( _ => None, } } + #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_dependency_job( job: &MiniPulledJob, @@ -251,6 +149,12 @@ pub async fn handle_dependency_job( token: &str, occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { + // Processing a dependency job - these jobs handle lockfile generation and dependency updates + // for scripts, flows, and apps when their dependencies or imported scripts change + tracing::debug!( + "Processing dependency job for path: {:?}", + job.runnable_path() + ); let script_path = job.runnable_path(); let raw_deps = job .args @@ -350,130 +254,25 @@ pub async fn handle_dependency_job( let current_hash = job.runnable_id.unwrap_or(ScriptHash(0)); let w_id = &job.workspace_id; + let (deployment_message, parent_path) = get_deployment_msg_and_parent_path_from_args(job.args.clone()); - let script_info = sqlx::query_as::<_, windmill_common::scripts::Script>( - "SELECT * FROM script WHERE hash = $1 AND workspace_id = $2", + // We do not create new row for this update + // That means we can keep current hash and just update lock + sqlx::query!( + "UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3", + &content, + ¤t_hash.0, + w_id ) - .bind(¤t_hash.0) - .bind(w_id) - .fetch_one(db) + .execute(db) .await?; - // DependencyJob can be triggered only from 2 places: - // 1. create_script function in windmill-api/src/scripts.rs - // 2. trigger_dependents_to_recompute_dependencies (in this file) - // - // First will **always** produce script with null in `lock` - // where Second will **always** do with lock being not null - let deployed_hash = if script_info.lock.is_some() && !*WMDEBUG_NO_HASH_CHANGE_ON_DJ { - let mut tx = db.begin().await?; - // This entire section exists to solve following problem: - // - // 2 workers, one script that depend on another in python - // run the original script on both workers - // you update the dependenecy of a relative import, - // run it again until you ran it on both, normally it should fail on one of those - // - // It happens because every worker has cached their own script versions. - // However usual dependency job does not update hash of the script (and cache is keyed by the hash). - // This logical branch will create new script which will update the hash and automatically invalidate cache. - // - // IMPORTANT: This will **only** be triggered by another DependencyJob. It will never be triggered by script (re)deployement - - let ns = NewScript { - path: script_info.path, - parent_hash: Some(current_hash), - summary: script_info.summary, - description: script_info.description, - content: script_info.content, - schema: script_info.schema, - is_template: Some(script_info.is_template), - // TODO: Make it either None everywhere (particularely when raw reqs are calculated) - // Or handle this case and conditionally make Some (only with raw reqs) - lock: None, - language: script_info.language, - kind: Some(script_info.kind), - tag: script_info.tag, - draft_only: script_info.draft_only, - envs: script_info.envs, - concurrent_limit: script_info.concurrent_limit, - concurrency_time_window_s: script_info.concurrency_time_window_s, - cache_ttl: script_info.cache_ttl, - dedicated_worker: script_info.dedicated_worker, - ws_error_handler_muted: script_info.ws_error_handler_muted, - priority: script_info.priority, - timeout: script_info.timeout, - delete_after_use: script_info.delete_after_use, - restart_unless_cancelled: script_info.restart_unless_cancelled, - deployment_message: deployment_message.clone(), - concurrency_key: script_info.concurrency_key, - visible_to_runner_only: script_info.visible_to_runner_only, - no_main_func: script_info.no_main_func, - codebase: script_info.codebase, - has_preprocessor: script_info.has_preprocessor, - on_behalf_of_email: script_info.on_behalf_of_email, - assets: script_info.assets, - }; - - let new_hash = hash_script(&ns); - - sqlx::query!(" - INSERT INTO script - (workspace_id, hash, path, parent_hashes, summary, description, content, \ - created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ - draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ - dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, \ - codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets) - - SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, \ - content, created_by, schema, is_template, extra_perms, $4, language, kind, tag, \ - draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ - dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, \ - codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets - - FROM script WHERE hash = $2 AND workspace_id = $3; - ", - new_hash, current_hash.0, w_id, &content).execute(db).await?; - - // Archive current - sqlx::query!( - "UPDATE script SET archived = true WHERE hash = $1 AND workspace_id = $2", - current_hash.0, - w_id - ) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - ScriptHash(new_hash) - } else { - // We do not create new row for this update - // That means we can keep current hash and just update lock - sqlx::query!( - "UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3", - &content, - ¤t_hash.0, - w_id - ) - .execute(db) - .await?; - - // `lock` has been updated; invalidate the cache. - // Since only worker that ran this Dependency Job has the cache - // we do not need to think about invalidating cache for other workers. - cache::script::invalidate(current_hash); - - if *WMDEBUG_NO_HASH_CHANGE_ON_DJ { - tracing::warn!("WMDEBUG_NO_HASH_CHANGE_ON_DJ usually should not be used. Behavior might be unstable. Please contact Windmill Team for support.") - } - - current_hash - }; + // `lock` has been updated; invalidate the cache. + // Since only worker that ran this Dependency Job has the cache + // we do not need to think about invalidating cache for other workers. + cache::script::invalidate(current_hash); if let Err(e) = handle_deployment_metadata( &job.permissioned_as_email, @@ -481,7 +280,7 @@ pub async fn handle_dependency_job( &db, &w_id, DeployedObject::Script { - hash: deployed_hash, + hash: current_hash, path: script_path.to_string(), parent_path: parent_path.clone(), }, @@ -548,7 +347,7 @@ fn remove_ansi_codes(s: &str) -> String { pub async fn process_relative_imports( db: &sqlx::Pool, - job_id: Option, + _job_id: Option, args: Option<&Json>>>, w_id: &str, script_path: &str, @@ -561,40 +360,50 @@ pub async fn process_relative_imports( permissioned_as: &str, lock: Option, ) -> error::Result<()> { - let relative_imports = extract_relative_imports(&code, script_path, script_lang); - if let Some(relative_imports) = relative_imports { - if (script_lang.is_some_and(|v| v == ScriptLang::Bun) - && lock - .as_ref() - .is_some_and(|v| v.contains("generatedFromPackageJson"))) - || (script_lang.is_some_and(|v| v == ScriptLang::Python3) + // TODO: Should be moved into handle_dependency_job body to be more consistent with how flows and apps are handled + { + let relative_imports = extract_relative_imports(&code, script_path, script_lang); + if let Some(relative_imports) = relative_imports { + let mut tx = db.begin().await?; + let mut dependency_map = ScopedDependencyMap::fetch_maybe_rearranged( + &w_id, + script_path, + "script", + &parent_path, + db, + ) + .await?; + if (script_lang.is_some_and(|v| v == ScriptLang::Bun) && lock .as_ref() - .is_some_and(|v| v.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT))) - { - // if the lock file is generated from a package.json/requirements.txt, we need to clear the dependency map - // because we do not want to have dependencies be recomputed automatically. Empty relative imports passed - // to update_script_dependency_map will clear the dependency map. - update_script_dependency_map( - &job_id.unwrap_or_else(|| Uuid::nil()), - db, - w_id, - &parent_path, - script_path, - vec![], - ) - .await?; - } else { - update_script_dependency_map( - &job_id.unwrap_or_else(|| Uuid::nil()), - db, - w_id, - &parent_path, - script_path, - relative_imports, - ) - .await?; + .is_some_and(|v| v.contains("generatedFromPackageJson"))) + || (script_lang.is_some_and(|v| v == ScriptLang::Python3) + && lock + .as_ref() + .is_some_and(|v| v.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT))) + { + // if the lock file is generated from a package.json/requirements.txt, we need to clear the dependency map + // because we do not want to have dependencies be recomputed automatically. Empty relative imports passed + // to update_script_dependency_map will clear the dependency map. + + // TODO: Rework the logic for synchronized raw requirements PR. + // For now we will just do nothing and let dissolve clear every item related to this script. + } else { + tx = dependency_map + .patch( + Some(relative_imports), + // Ideally should be None, but due to current implementation will use empty string to represent None. + "".into(), + tx, + ) + .await?; + } + // If felt into first branch which did not call .patch(, this operation will clean dependency_map for this script. + dependency_map.dissolve(tx).await.commit().await?; } + } + + { let already_visited = args .map(|x| { x.get("already_visited") @@ -603,25 +412,50 @@ pub async fn process_relative_imports( }) .flatten() .unwrap_or_default(); - if let Err(e) = trigger_dependents_to_recompute_dependencies( - w_id, - script_path, - deployment_message, - parent_path, - permissioned_as_email, - created_by, - permissioned_as, - db, - already_visited, + + // But currently we will do this extra db call for every script regardless of whether they have relative imports or not + // Script might have no relative imports but still be referenced by someone else. + match timeout( + core::time::Duration::from_secs(60), + Box::pin(trigger_dependents_to_recompute_dependencies( + w_id, + script_path, + deployment_message, + parent_path, + permissioned_as_email, + created_by, + permissioned_as, + db, + already_visited, + )), ) + .warn_after_seconds(10) .await { - tracing::error!(%e, "error triggering dependents to recompute dependencies"); + Ok(Err(e)) => { + tracing::error!(%e, "error triggering dependents to recompute dependencies") + } + Err(e) => { + tracing::error!(%e, "triggering dependents to recompute dependencies has timed out") + } + _ => {} } } + Ok(()) } +pub fn is_generated_from_raw_requirements(lang: Option, lock: &Option) -> bool { + (lang.is_some_and(|v| v == ScriptLang::Bun) + && lock + .as_ref() + .is_some_and(|v| v.contains("generatedFromPackageJson"))) + || (lang.is_some_and(|v| v == ScriptLang::Python3) + && lock + .as_ref() + .is_some_and(|v| v.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT))) +} + pub async fn trigger_dependents_to_recompute_dependencies( w_id: &str, script_path: &str, @@ -633,6 +467,15 @@ pub async fn trigger_dependents_to_recompute_dependencies( db: &sqlx::Pool, mut already_visited: Vec, ) -> error::Result<()> { + // TODO: There is a race-condition. + // This can be old version. + // + // Check lines of code below, you will find that we get the latest version of the script/app/flow + // + // However the latest version does not necessarily mean that it is finalized. + // Instead we assume that this would be the version we would base on. + // + // So the script_importers might be behind. Thus some information like nodes_to_relock might be lost. let script_importers = sqlx::query!( "SELECT importer_path, importer_kind::text, array_agg(importer_node_id) as importer_node_ids FROM dependency_map WHERE imported_path = $1 @@ -644,13 +487,20 @@ pub async fn trigger_dependents_to_recompute_dependencies( .fetch_all(db) .await?; + tracing::debug!( + "Triggering dependents to recompute dependencies for: {}", + &script_path + ); + already_visited.push(script_path.to_string()); for s in script_importers.iter() { + tracing::trace!("Processing dependency: {:?}", &s); if already_visited.contains(&s.importer_path) { + tracing::trace!("Skipping already visited dependency"); continue; } - let tx = PushIsolationLevel::IsolatedRoot(db.clone()); + let mut tx = db.clone().begin().await?; let mut args: HashMap> = HashMap::new(); if let Some(ref dm) = deployment_message { args.insert("deployment_message".to_string(), to_raw_value(&dm)); @@ -668,192 +518,134 @@ pub async fn trigger_dependents_to_recompute_dependencies( to_raw_value(&already_visited), ); + args.insert( + "triggered_by_relative_import".to_string(), + to_raw_value(&()), + ); + + // Lock the debounce_key entry FOR UPDATE to coordinate with the push side. + // This prevents concurrent modifications during dependency job scheduling. + // + // The lock serves two purposes: + // 1. Ensures we get the current debounce_job_id atomically + // 2. Blocks new push requests from modifying this key until we commit + // 3. Blocks puller from actually starting the job and gives us a chance to still squeeze the debounce in. + // + // After our transaction commits, any pending push/pull requests can proceed with + // their debounce logic. + let debounce_job_id_o = + windmill_common::jobs::lock_debounce_key(w_id, &s.importer_path, &mut tx).await?; + + tracing::debug!( + debounce_job_id = ?debounce_job_id_o, + importer_path = %s.importer_path, + "Retrieved debounce job ID (if exists)" + ); + let kind = s.importer_kind.clone().unwrap_or_default(); let job_payload = if kind == "script" { - let r = - // TODO: Not sure if this is safe: - // might have race conditions in edge-cases - get_latest_deployed_hash_for_path(None, db.clone(), w_id, s.importer_path.as_str()) - .await; - match r { - // We will create Dependency job as is. But the Dep Job Handler will detect that the job originates - // from [[trigger_dependents_to_recompute_dependencies]] and will create new script with new hash instead - Ok(r) => JobPayload::Dependencies { - path: s.importer_path.clone(), - hash: ScriptHash(r.hash), - language: r.language, - dedicated_worker: r.dedicated_worker, - }, - Err(err) => { - tracing::error!( - "error getting latest deployed hash for path {path}: {err}", - path = s.importer_path, - err = err - ); + // TODO: Make it query only non-archived + match sqlx::query_scalar!( + "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false ORDER BY created_at DESC LIMIT 1", + s.importer_path.clone(), + w_id + ) + .fetch_optional(&mut *tx) + .await? + { + Some(hash) => { + tracing::debug!("newest hash for {} is: {hash}", &s.importer_path); + + let info = + windmill_common::get_script_info_for_hash(None, db, w_id, hash).await?; + + JobPayload::Dependencies { + path: s.importer_path.clone(), + hash: ScriptHash(hash), + language: info.language, + dedicated_worker: info.dedicated_worker, + } + } + None => { + ScopedDependencyMap::clear_map_for_item( + &s.importer_path, + w_id, + "script", + tx, + &None, + ) + .await + .commit() + .await?; continue; } } } else if kind == "flow" { - // Unlike 'script', 'flow' will not delegate redeployment of new flow to the Dep Job Handler. - // We will create new flow in-place. - // It would be harder to do otherwise. + tracing::debug!("Handling flow dependency update for: {}", s.importer_path); - // Create transaction to make operation atomic. - let mut flow_tx = db.begin().await?; args.insert( "nodes_to_relock".to_string(), to_raw_value(&s.importer_node_ids), ); - let r = sqlx::query_scalar!( - "SELECT versions[array_upper(versions, 1)] FROM flow WHERE path = $1 AND workspace_id = $2", - s.importer_path, - w_id, - ).fetch_one(&mut *flow_tx) - .await - .map_err(to_anyhow); - - match r { - // TODO: Fallback - remove eventually. - Ok(Some(version)) if *WMDEBUG_NO_NEW_FLOW_VERSION_ON_DJ => { - tracing::warn!("WMDEBUG_NO_NEW_FLOW_VERSION_ON_DJ usually should not be used. Behavior might be unstable. Please contact Windmill Team for support."); - JobPayload::FlowDependencies { - path: s.importer_path.clone(), - dedicated_worker: None, - version, - } - } - // Get current version of current flow. - Ok(Some(cur_version)) => { - // NOTE: Temporary solution. See the usage for more details. - args.insert( - "triggered_by_relative_import".to_string(), - to_raw_value(&()), - ); - // Find out what would be the next version. - // Also clone current flow_version to get new_version (which is usually c_v + 1). - // NOTE: It is fine if something goes wrong downstream and `flow` is not being appended with this new version. - // This version will just remain in db and cause no trouble. - let new_version = sqlx::query_scalar!( - "INSERT INTO flow_version - (workspace_id, path, value, schema, created_by) - - SELECT workspace_id, path, value, schema, created_by - FROM flow_version WHERE path = $1 AND workspace_id = $2 AND id = $3 - - RETURNING id", + match sqlx::query_scalar!( + "SELECT id FROM flow_version WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", + s.importer_path.clone(), + w_id + ) + .fetch_optional(&mut *tx) + .await? + { + Some(version) => JobPayload::FlowDependencies { + path: s.importer_path.clone(), + version, + dedicated_worker: None, + }, + None => { + ScopedDependencyMap::clear_map_for_item( &s.importer_path, w_id, - cur_version + "flow", + tx, + &None, ) - .fetch_one(&mut *flow_tx) .await - .map_err(|e| { - error::Error::internal_err(format!( - "Error updating flow due to flow history insert: {e:#}" - )) - })?; - - // Commit the transaction. - // NOTE: - // We do not append flow.versions with new version. - // We will do this in the end of the dependency job handler. - // Otherwise it might become a source of race-conditions. - flow_tx.commit().await?; - JobPayload::FlowDependencies { - path: s.importer_path.clone(), - dedicated_worker: None, - // Point Dep Job to the new version. - // We do this since we want to assume old ones are immutable. - version: new_version, - } - } - Ok(None) => { - tracing::error!( - "no flow version found for path {path}", - path = s.importer_path - ); - // Do not commit the transaction. It will be dropped and rollbacked - continue; - } - Err(err) => { - tracing::error!( - "error getting latest deployed flow version for path {path}: {err}", - path = s.importer_path, - ); - // Do not commit the transaction. It will be dropped and rollbacked + .commit() + .await?; continue; } } } else if kind == "app" && !*WMDEBUG_NO_NEW_APP_VERSION_ON_DJ { - // Create transaction to make operation atomic. - let mut tx = db.begin().await?; + tracing::debug!("Handling flow dependency update for: {}", s.importer_path); args.insert( "components_to_relock".to_string(), + // TODO: unsafe. Importer Node Ids are not checked. They can simply be array of empty strings! to_raw_value(&s.importer_node_ids), ); - let r = sqlx::query_scalar!( - "SELECT versions[array_upper(versions, 1)] FROM app WHERE path = $1 AND workspace_id = $2", - s.importer_path, - w_id, - ).fetch_one(&mut *tx) - .await - .map_err(to_anyhow); - - match r { - // Get current version of current flow. - Ok(Some(cur_version)) => { - // NOTE: Temporary solution. See the usage for more details. - args.insert( - "triggered_by_relative_import".to_string(), - to_raw_value(&()), - ); - - let new_version = sqlx::query_scalar!( - "INSERT INTO app_version - (app_id, value, created_by, raw_app) - SELECT app_id, value, created_by, raw_app - FROM app_version WHERE id = $1 - RETURNING id", - cur_version + match sqlx::query_scalar!( + "SELECT id FROM app_version WHERE app_id = (SELECT id FROM app WHERE path = $1 AND workspace_id = $2) ORDER BY created_at DESC LIMIT 1", + s.importer_path.clone(), + w_id + ) + .fetch_optional(&mut *tx) + .await? + { + Some(version) => { + JobPayload::AppDependencies { path: s.importer_path.clone(), version } + } + None => { + ScopedDependencyMap::clear_map_for_item( + &s.importer_path, + w_id, + "app", + tx, + &None, ) - .fetch_one(&mut *tx) .await - .map_err(|e| { - error::Error::internal_err(format!( - "Error updating App due to App history insert: {e:#}" - )) - })?; - - // Commit the transaction. - // NOTE: - // We do not append app.versions with new version. - // We will do this in the end of the dependency job handler. - // Otherwise it might become a source of race-conditions. - tx.commit().await?; - JobPayload::AppDependencies { - path: s.importer_path.clone(), - // Point Dep Job to the new version. - // We do this since we want to assume old ones are immutable. - version: new_version, - } - } - Ok(None) => { - tracing::error!( - "no app version found for path {path}", - path = s.importer_path - ); - // Do not commit the transaction. It will be dropped and rollbacked - continue; - } - Err(err) => { - tracing::error!( - "error getting latest deployed app version for path {path}: {err}", - path = s.importer_path, - ); - // Do not commit the transaction. It will be dropped and rollbacked + .commit() + .await?; continue; } } @@ -866,9 +658,10 @@ pub async fn trigger_dependents_to_recompute_dependencies( continue; }; + tracing::debug!("Pushing dependency job for: {}", s.importer_path); let (job_uuid, new_tx) = windmill_queue::push( db, - tx, + PushIsolationLevel::Transaction(tx), &w_id, job_payload, windmill_queue::PushArgs { args: &args, extra: None }, @@ -876,7 +669,8 @@ pub async fn trigger_dependents_to_recompute_dependencies( email, permissioned_as.to_string(), Some("trigger.dependents.to.recompute.dependencies"), - None, + // Schedule for future for debouncing. + Some(Utc::now() + Duration::seconds(*DEPENDENCY_JOB_DEBOUNCE_DELAY as i64)), None, None, None, @@ -886,14 +680,17 @@ pub async fn trigger_dependents_to_recompute_dependencies( false, None, true, - None, + Some("dependency".into()), None, None, None, None, false, + None, + debounce_job_id_o, ) .await?; + tracing::info!( "pushed dependency job due to common python path: {job_uuid} for path {path}", path = s.importer_path, @@ -904,7 +701,7 @@ pub async fn trigger_dependents_to_recompute_dependencies( } pub async fn handle_flow_dependency_job( - job: &MiniPulledJob, + job: MiniPulledJob, preview_data: Option<&RawData>, mem_peak: &mut i32, canceled_by: &mut Option, @@ -916,6 +713,9 @@ pub async fn handle_flow_dependency_job( token: &str, occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { + tracing::debug!("Processing flow dependency job"); + tracing::trace!("Job details: {:?}", &job); + tracing::trace!("Preview data: {:?}", &preview_data); let job_path = job.runnable_path.clone().ok_or_else(|| { error::Error::internal_err( "Cannot resolve flow dependencies for flow without path".to_string(), @@ -933,6 +733,12 @@ pub async fn handle_flow_dependency_job( .flatten() .unwrap_or(false); + let triggered_by_relative_import = job + .args + .as_ref() + .map(|x| x.get("triggered_by_relative_import").is_some()) + .unwrap_or_default(); + let version = if skip_flow_update { None } else { @@ -948,6 +754,7 @@ pub async fn handle_flow_dependency_job( ) }; + tracing::trace!("Job details: {:?}", &job); let (deployment_message, parent_path) = get_deployment_msg_and_parent_path_from_args(job.args.clone()); @@ -961,6 +768,7 @@ pub async fn handle_flow_dependency_job( }) .flatten(); + tracing::debug!("Nodes to relock: {:?}", &nodes_to_relock); let raw_deps = job .args .as_ref() @@ -971,12 +779,6 @@ pub async fn handle_flow_dependency_job( }) .flatten(); - let triggered_by_relative_import = job - .args - .as_ref() - .map(|x| x.get("triggered_by_relative_import").is_some()) - .unwrap_or_default(); - // `JobKind::FlowDependencies` job store either: // - A saved flow version `id` in the `script_hash` column. // - Preview raw flow in the `queue` or `job` table. @@ -991,8 +793,15 @@ pub async fn handle_flow_dependency_job( .clone(); let mut tx = db.begin().await?; - tx = clear_dependency_parent_path(&parent_path, &job_path, &job.workspace_id, "flow", tx) - .await?; + + let mut dependency_map = ScopedDependencyMap::fetch_maybe_rearranged( + &job.workspace_id, + &job_path, + "flow", + &parent_path, + &mut *tx, + ) + .await?; if !skip_flow_update { sqlx::query!( @@ -1008,9 +817,9 @@ pub async fn handle_flow_dependency_job( let modified_ids; let errors; - (flow.modules, tx, modified_ids, errors) = lock_modules( - flow.modules, - job, + (flow, tx, modified_ids, errors) = lock_flow_value( + flow, + &job, mem_peak, canceled_by, job_dir, @@ -1024,7 +833,8 @@ pub async fn handle_flow_dependency_job( &nodes_to_relock, occupancy_metrics, skip_flow_update, - raw_deps, + &raw_deps, + &mut dependency_map, ) .await?; @@ -1082,6 +892,7 @@ pub async fn handle_flow_dependency_job( tracing::error!(%job.id, %err, "error checking cancellation for job {0}: {err}", job.id); false }) { + // Drop tx and thus cancel any changes return Ok(to_raw_value_owned(json!({ "status": "Flow lock generation was canceled", }))); @@ -1092,6 +903,8 @@ pub async fn handle_flow_dependency_job( Error::internal_err("Flow Dependency requires script hash (flow version)".to_owned()) })?; + tx = dependency_map.dissolve(tx).await; + sqlx::query!( "UPDATE flow SET value = $1 WHERE path = $2 AND workspace_id = $3", &new_flow_value as &Json>, @@ -1110,6 +923,7 @@ pub async fn handle_flow_dependency_job( // Compute a lite version of the flow value (`RawScript` => `FlowScript`). let mut value_lite = flow.clone(); + tx = reduce_flow( tx, &mut value_lite.modules, @@ -1140,6 +954,8 @@ pub async fn handle_flow_dependency_job( &job_path, &job.workspace_id, ).execute(&mut *tx).await?; + tracing::debug!("Marked flow version as latest"); + tracing::debug!("Flow version: {}", version); } tx.commit().await?; @@ -1196,6 +1012,135 @@ struct LockModuleError { error: Error, } +// Process entire FlowValue including failure_module and preprocessor_module +async fn lock_flow_value<'c>( + mut flow: FlowValue, + job: &MiniPulledJob, + mem_peak: &mut i32, + canceled_by: &mut Option, + job_dir: &str, + db: &sqlx::Pool, + mut tx: sqlx::Transaction<'c, sqlx::Postgres>, + worker_name: &str, + worker_dir: &str, + job_path: &str, + base_internal_url: &str, + token: &str, + locks_to_reload: &Option>, + occupancy_metrics: &mut OccupancyMetrics, + skip_flow_update: bool, + raw_deps: &Option>, + dependency_map: &mut ScopedDependencyMap, +) -> Result<( + FlowValue, + sqlx::Transaction<'c, sqlx::Postgres>, + Vec, + Vec, +)> { + let mut all_modified_ids = Vec::new(); + let mut all_errors = Vec::new(); + + // Process main modules + let (updated_modules, updated_tx, modules_modified_ids, modules_errors) = lock_modules( + flow.modules, + job, + mem_peak, + canceled_by, + job_dir, + db, + tx, + worker_name, + worker_dir, + job_path, + base_internal_url, + token, + locks_to_reload, + occupancy_metrics, + skip_flow_update, + &raw_deps, + dependency_map, + ) + .await?; + + tx = updated_tx; + flow.modules = updated_modules; + all_modified_ids.extend(modules_modified_ids); + all_errors.extend(modules_errors); + + // Process failure_module if it exists + if let Some(failure_module) = flow.failure_module { + let (updated_failure_modules, updated_tx, failure_modified_ids, failure_errors) = + lock_modules( + vec![*failure_module], + job, + mem_peak, + canceled_by, + job_dir, + db, + tx, + worker_name, + worker_dir, + job_path, + base_internal_url, + token, + locks_to_reload, + occupancy_metrics, + skip_flow_update, + &raw_deps, + dependency_map, + ) + .await?; + + tx = updated_tx; + all_modified_ids.extend(failure_modified_ids); + all_errors.extend(failure_errors); + + flow.failure_module = updated_failure_modules.into_iter().next().map(Box::new); + } + + // Process preprocessor_module if it exists + if let Some(preprocessor_module) = flow.preprocessor_module { + let ( + updated_preprocessor_modules, + updated_tx, + preprocessor_modified_ids, + preprocessor_errors, + ) = lock_modules( + vec![*preprocessor_module], + job, + mem_peak, + canceled_by, + job_dir, + db, + tx, + worker_name, + worker_dir, + job_path, + base_internal_url, + token, + locks_to_reload, + occupancy_metrics, + skip_flow_update, + &raw_deps, + dependency_map, + ) + .await?; + + tx = updated_tx; + all_modified_ids.extend(preprocessor_modified_ids); + all_errors.extend(preprocessor_errors); + + flow.preprocessor_module = updated_preprocessor_modules + .into_iter() + .next() + .map(Box::new); + } + + Ok((flow, tx, all_modified_ids, all_errors)) +} + +// TODO: Maybe use [FlowValue::traverse_leafs] +// IMPORTANT: If updating this function, make sure you also update [FlowValue::traverse_leafs] async fn lock_modules<'c>( modules: Vec, job: &MiniPulledJob, @@ -1212,8 +1157,8 @@ async fn lock_modules<'c>( locks_to_reload: &Option>, occupancy_metrics: &mut OccupancyMetrics, skip_flow_update: bool, - raw_deps: Option>, - // (modules to replace old seq (even unmmodified ones), new transaction, modified ids) ) + raw_deps: &Option>, + dependency_map: &mut ScopedDependencyMap, // (modules to replace old seq (even unmmodified ones), new transaction, modified ids) ) ) -> Result<( Vec, sqlx::Transaction<'c, sqlx::Postgres>, @@ -1224,8 +1169,6 @@ async fn lock_modules<'c>( let mut modified_ids = Vec::new(); let mut errors = Vec::new(); for mut e in modules.into_iter() { - let id = e.id.clone(); - let mut nmodified_ids = Vec::new(); let FlowModuleValue::RawScript { lock, path, @@ -1240,6 +1183,8 @@ async fn lock_modules<'c>( assets, } = e.get_value()? else { + let mut nmodified_ids = Vec::new(); + let mut nerrors = Vec::new(); match e.get_value()? { FlowModuleValue::ForloopFlow { iterator, @@ -1250,7 +1195,7 @@ async fn lock_modules<'c>( parallelism, } => { let nmodules; - (nmodules, tx, modified_ids, errors) = Box::pin(lock_modules( + (nmodules, tx, nmodified_ids, nerrors) = Box::pin(lock_modules( modules, job, mem_peak, @@ -1266,7 +1211,8 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, - raw_deps.clone(), + &raw_deps, + dependency_map, )) .await?; e.value = FlowModuleValue::ForloopFlow { @@ -1281,7 +1227,6 @@ async fn lock_modules<'c>( } FlowModuleValue::BranchAll { branches, parallel } => { let mut nbranches = vec![]; - nmodified_ids = vec![]; for mut b in branches { let nmodules; let inner_modified_ids; @@ -1302,7 +1247,8 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, - raw_deps.clone(), + &raw_deps, + dependency_map, )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -1314,7 +1260,7 @@ async fn lock_modules<'c>( } FlowModuleValue::WhileloopFlow { modules, modules_node, skip_failures } => { let nmodules; - (nmodules, tx, nmodified_ids, errors) = Box::pin(lock_modules( + (nmodules, tx, nmodified_ids, nerrors) = Box::pin(lock_modules( modules, job, mem_peak, @@ -1330,7 +1276,8 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, - raw_deps.clone(), + &raw_deps, + dependency_map, )) .await?; e.value = FlowModuleValue::WhileloopFlow { @@ -1342,7 +1289,6 @@ async fn lock_modules<'c>( } FlowModuleValue::BranchOne { branches, default, default_node } => { let mut nbranches = vec![]; - nmodified_ids = vec![]; for mut b in branches { let nmodules; let inner_modified_ids; @@ -1363,7 +1309,8 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, - raw_deps.clone(), + &raw_deps, + dependency_map, )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -1373,7 +1320,8 @@ async fn lock_modules<'c>( } let ndefault; let ninner_errors; - (ndefault, tx, nmodified_ids, ninner_errors) = Box::pin(lock_modules( + let ninner_modified_ids; + (ndefault, tx, ninner_modified_ids, ninner_errors) = Box::pin(lock_modules( default, job, mem_peak, @@ -1389,10 +1337,12 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, - raw_deps.clone(), + &raw_deps, + dependency_map, )) .await?; errors.extend(ninner_errors); + nmodified_ids.extend(ninner_modified_ids); e.value = FlowModuleValue::BranchOne { branches: nbranches, default: ndefault, @@ -1423,9 +1373,57 @@ async fn lock_modules<'c>( .execute(&mut *tx) .await?; } + FlowModuleValue::AIAgent { input_transforms, mut tools } => { + // Extract FlowModules from tools and track their original indices + // MCP tools don't need locking, so we filter them out + let mut flow_modules = Vec::new(); + let mut flow_module_indices = Vec::new(); + + for (idx, tool) in tools.iter().enumerate() { + if let Some(flow_module) = Option::::from(tool) { + // Convert AgentTool -> FlowModule for locking + flow_modules.push(flow_module); + flow_module_indices.push(idx); + } + } + + // Lock only the FlowModule-type tools + let locked_flow_modules; + (locked_flow_modules, tx, nmodified_ids, nerrors) = Box::pin(lock_modules( + flow_modules, + job, + mem_peak, + canceled_by, + job_dir, + db, + tx, + worker_name, + worker_dir, + job_path, + base_internal_url, + token, + locks_to_reload, + occupancy_metrics, + skip_flow_update, + &raw_deps, + dependency_map, + )) + .await?; + + let mut locked_iter = locked_flow_modules.into_iter(); + for idx in flow_module_indices { + let locked = locked_iter.next().ok_or_else(|| { + Error::internal_err("locked tool module should exist".to_string()) + })?; + tools[idx] = locked.into(); + } + + e.value = FlowModuleValue::AIAgent { input_transforms, tools }.into(); + } _ => (), }; modified_ids.extend(nmodified_ids); + errors.extend(nerrors); new_flow_modules.push(e); continue; }; @@ -1441,8 +1439,23 @@ async fn lock_modules<'c>( .await?; } + let get_imports = || { + let dep_path = path.clone().unwrap_or_else(|| job_path.to_string()); + extract_relative_imports( + &content, + &format!("{dep_path}/flow"), + &Some(language.clone()), + ) + }; + if let Some(locks_to_reload) = locks_to_reload { if !locks_to_reload.contains(&e.id) { + if !is_generated_from_raw_requirements(Some(language), &lock) { + let relative_imports = get_imports(); + tx = dependency_map + .patch(relative_imports.clone(), e.id.clone(), tx) + .await?; + } new_flow_modules.push(e); continue; } @@ -1450,6 +1463,13 @@ async fn lock_modules<'c>( if lock.as_ref().is_some_and(|x| !x.trim().is_empty()) { let skip_creating_new_lock = skip_creating_new_lock(&language, &content); if skip_creating_new_lock { + if !is_generated_from_raw_requirements(Some(language), &lock) { + let relative_imports = get_imports(); + tx = dependency_map + .patch(relative_imports.clone(), e.id.clone(), tx) + .await?; + } + new_flow_modules.push(e); continue; } @@ -1466,16 +1486,16 @@ async fn lock_modules<'c>( })?; // If we have local lockfiles (and they are enabled) we will replace script content with lockfile and tell hander that it is raw_deps job - let (content, raw_deps) = raw_deps + let (content_for_capture, raw_deps) = raw_deps .as_ref() .and_then(|llfs| llfs.get(language.as_str())) .map(|lock| (lock.to_owned(), true)) - .unwrap_or((content, false)); + .unwrap_or((content.clone(), false)); let new_lock = capture_dependency_job( &job.id, &language, - &content, + &content_for_capture, mem_peak, canceled_by, job_dir, @@ -1497,37 +1517,12 @@ async fn lock_modules<'c>( // let lock = match new_lock { Ok(new_lock) => { - let dep_path = path.clone().unwrap_or_else(|| job_path.to_string()); - tx = clear_dependency_map_for_item( - &job_path, - &job.workspace_id, - "flow", - tx, - &Some(e.id.clone()), - ) - .await?; - let relative_imports = extract_relative_imports( - &content, - &format!("{dep_path}/flow"), - &Some(language.clone()), - ); - if let Some(relative_imports) = relative_imports { - let mut logs = "".to_string(); - logs.push_str(format!("\n\n--- RELATIVE IMPORTS of {} ---\n\n", e.id).as_str()); - - tx = add_relative_imports_to_dependency_map( - &dep_path, - &job.workspace_id, - relative_imports, - "flow", - tx, - &mut logs, - Some(e.id.clone()), - ) - .await?; - append_logs(&job.id, &job.workspace_id, logs, &db.into()).await; + if !raw_deps && !skip_flow_update { + let relative_imports = get_imports(); + tx = dependency_map + .patch(relative_imports.clone(), e.id.clone(), tx) + .await?; } - if language == ScriptLang::Bun || language == ScriptLang::Bunnative { let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content); if anns.native && language == ScriptLang::Bun { @@ -1536,11 +1531,12 @@ async fn lock_modules<'c>( language = ScriptLang::Bun; }; } + Some(new_lock) } Err(error) => { // TODO: Record flow raw script error lock logs - errors.push(LockModuleError { id, error }); + errors.push(LockModuleError { id: e.id.clone(), error }); None } }; @@ -1637,7 +1633,6 @@ async fn insert_flow_node<'c>( Ok((tx, FlowNodeId(id))) } -// TODO: Clean up dependency map when moved/renamed? async fn insert_app_script( db: &sqlx::Pool, path: &str, @@ -1770,6 +1765,7 @@ async fn reduce_flow<'c>( Some(language), ) .await?; + val = FlowScript { input_transforms, id, @@ -1905,6 +1901,10 @@ fn skip_creating_new_lock(language: &ScriptLang, content: &str) -> bool { true } +// TODO: Use transaction? +// TODO: Use abstracted traverse function. +// +// IMPORTANT: If updating this function, make sure you also update [traverse_app_inline_scripts] #[async_recursion] async fn lock_modules_app( value: Value, @@ -1922,6 +1922,7 @@ async fn lock_modules_app( locks_to_reload: &Option>, // Represents the closest container id container_id: Option, + dependency_map: &mut ScopedDependencyMap, ) -> Result { match value { Value::Object(mut m) => { @@ -1956,6 +1957,12 @@ async fn lock_modules_app( .to_string(); let mut logs = "".to_string(); + let relative_imports = extract_relative_imports( + &content, + &format!("{job_path}/app"), + &Some(language.clone()), + ); + if let Some((l, id)) = locks_to_reload .as_ref() .zip(container_id.as_ref()) @@ -1969,6 +1976,15 @@ async fn lock_modules_app( }) { if !l.contains(id) { + dependency_map + .patch( + relative_imports.clone(), + container_id.unwrap_or_default(), + db.begin().await?, + ) + .await? + .commit() + .await?; return Ok(Value::Object(m.clone())); } } else if v @@ -1976,6 +1992,16 @@ async fn lock_modules_app( .is_some_and(|x| !x.as_str().unwrap().trim().is_empty()) { if skip_creating_new_lock(&language, &content) { + dependency_map + .patch( + relative_imports.clone(), + container_id.unwrap_or_default(), + db.begin().await?, + ) + .await? + .commit() + .await?; + logs.push_str( "Found already locked inline script. Skipping lock...\n", ); @@ -2006,44 +2032,15 @@ async fn lock_modules_app( Ok(new_lock) => { append_logs(&job.id, &job.workspace_id, logs, &db.into()).await; - let mut tx = db.begin().await?; - - tx = clear_dependency_map_for_item( - &job_path, - &job.workspace_id, - "app", - tx, - &container_id, - ) - .await?; - - let relative_imports = extract_relative_imports( - &content, - &format!("{job_path}/app"), - &Some(language.clone()), - ); - - if let Some(relative_imports) = relative_imports { - let mut logs = "".to_string(); - logs.push_str( - format!("\n\n--- RELATIVE IMPORTS ---\n\n").as_str(), - ); - - tx = add_relative_imports_to_dependency_map( - &job_path, - &job.workspace_id, - relative_imports, - "app", - tx, - &mut logs, - container_id, + dependency_map + .patch( + relative_imports.clone(), + container_id.unwrap_or_default(), + db.begin().await?, ) + .await? + .commit() .await?; - append_logs(&job.id, &job.workspace_id, logs, &db.into()) - .await; - } - - tx.commit().await?; let anns = windmill_common::worker::TypeScriptAnnotations::parse( @@ -2103,6 +2100,7 @@ async fn lock_modules_app( .and_then(Value::as_str) .map(str::to_owned) .or(container_id.clone()), + dependency_map, ) .await?, ); @@ -2128,6 +2126,7 @@ async fn lock_modules_app( occupancy_metrics, locks_to_reload, container_id.clone(), + dependency_map, ) .await?, ); @@ -2139,7 +2138,7 @@ async fn lock_modules_app( } pub async fn handle_app_dependency_job( - job: &MiniPulledJob, + job: MiniPulledJob, mem_peak: &mut i32, canceled_by: &mut Option, job_dir: &str, @@ -2191,11 +2190,22 @@ pub async fn handle_app_dependency_job( .await? .map(|record| (record.app_id, record.value)); + let (_, parent_path) = get_deployment_msg_and_parent_path_from_args(job.args.clone()); + + let mut dependency_map = ScopedDependencyMap::fetch_maybe_rearranged( + &job.workspace_id, + &job_path, + "app", + &parent_path, + db, + ) + .await?; + // TODO: Use transaction for entire segment? if let Some((app_id, value)) = record { let value = lock_modules_app( value, - job, + &job, mem_peak, canceled_by, job_dir, @@ -2208,9 +2218,17 @@ pub async fn handle_app_dependency_job( occupancy_metrics, &components_to_relock, None, + &mut dependency_map, ) .await?; + // TODO: Dissolve in the end? + dependency_map + .dissolve(db.begin().await?) + .await + .commit() + .await?; + // Compute a lite version of the app value (w/ `inlineScript.{lock,code}`). let mut value_lite = value.clone(); reduce_app(db, &job_path, &mut value_lite, app_id).await?; diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index d654e5af65..366cc9cdf7 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -308,7 +308,7 @@ pub(crate) async fn queue_vacuum(conn: &Connection, worker_name: &str, hostname: tokio::task::spawn( (async move { tracing::info!(worker = %worker_name, hostname = %hostname, "vacuuming queue"); - if let Err(e) = sqlx::query!("VACUUM v2_job_queue, v2_job_runtime, v2_job_status, job_perms") + if let Err(e) = sqlx::query!("VACUUM (SKIP_LOCKED) v2_job_queue, v2_job_runtime, v2_job_status, job_perms") .execute(&db2) .await { diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index eb9258e533..d8ac299e57 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.542.1"; +export const VERSION = "v1.573.3"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/build.sh b/cli/build.sh index 0da1c343a3..9a86ccbb54 100755 --- a/cli/build.sh +++ b/cli/build.sh @@ -16,4 +16,5 @@ set -e echo "Running dnt..." deno run -A dnt.ts -echo "Build complete!" \ No newline at end of file +echo "Build complete!" + diff --git a/cli/src/commands/dev/dev.ts b/cli/src/commands/dev/dev.ts index 58d8d03b33..87ef3458d1 100644 --- a/cli/src/commands/dev/dev.ts +++ b/cli/src/commands/dev/dev.ts @@ -19,7 +19,7 @@ import { mergeConfigWithConfigFile, readConfigFile, } from "../../core/conf.ts"; -import { exts, findGlobalDeps, removeExtensionToPath } from "../script/script.ts"; +import { exts, removeExtensionToPath } from "../script/script.ts"; import { inferContentTypeFromFilePath } from "../../utils/script_common.ts"; import { OpenFlow } from "../../../gen/types.gen.ts"; import { FlowFile } from "../flow/flow.ts"; @@ -82,8 +82,8 @@ async function dev(opts: GlobalOptions & SyncOptions) { localPath, SEP, undefined, - (path: string, newPath: string) => Deno.renameSync(path, newPath), - (path: string) => Deno.removeSync(path), + // (path: string, newPath: string) => Deno.renameSync(path, newPath), + // (path: string) => Deno.removeSync(path), ); currentLastEdit = { type: "flow", @@ -97,13 +97,10 @@ async function dev(opts: GlobalOptions & SyncOptions) { const splitted = cpath.split("."); const wmPath = splitted[0]; const lang = inferContentTypeFromFilePath(cpath, conf.defaultTs); - const globalDeps = await findGlobalDeps(); const typed = (await parseMetadataFile( removeExtensionToPath(cpath), undefined, - globalDeps, - [] ) )?.payload diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 93938e6254..e16f2fbcb3 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -14,7 +14,6 @@ import { FSFSElement, elementsToMap, ignoreF } from "../sync/sync.ts"; import { Flow } from "../../../gen/types.gen.ts"; import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts"; - export interface FlowFile { summary: string; description?: string; @@ -55,7 +54,7 @@ export async function pushFlow( async (path: string) => await Deno.readTextFile(localPath + path), log, localPath, - SEP, + SEP ); if (flow) { @@ -191,7 +190,7 @@ async function run( workspace: workspace.workspaceId, id, }); - log.info(jobInfo.result ?? {}); + log.info(JSON.stringify(jobInfo.result ?? {}, null, 2)); } async function generateLocks( @@ -201,14 +200,23 @@ async function generateLocks( } & SyncOptions, folder: string | undefined ) { - const useRawReqs = opts.useRawRequirements || Deno.env.get("USE_RAW_REQUIREMENTS") === "true"; + const useRawReqs = + opts.useRawRequirements || Deno.env.get("USE_RAW_REQUIREMENTS") === "true"; const workspace = await resolveWorkspace(opts); await requireLogin(opts); opts = await mergeConfigWithConfigFile(opts); if (folder) { // read script metadata file - await generateFlowLockInternal(folder, false, workspace, opts, undefined, undefined, useRawReqs); + await generateFlowLockInternal( + folder, + false, + workspace, + opts, + undefined, + undefined, + useRawReqs + ); } else { const ignore = await ignoreF(opts); const elems = Object.keys( @@ -229,10 +237,18 @@ async function generateLocks( let hasAny = false; for (const folder of elems) { - const candidate = await generateFlowLockInternal(folder, true, workspace, opts, undefined, undefined, useRawReqs); + const candidate = await generateFlowLockInternal( + folder, + true, + workspace, + opts, + undefined, + undefined, + useRawReqs + ); if (candidate) { hasAny = true; - log.info(colors.green(`+ ${candidate}`)); + log.info(colors.yellow.bold(`~ ${candidate}`)); } } @@ -251,7 +267,15 @@ async function generateLocks( return; } for (const folder of elems) { - await generateFlowLockInternal(folder, false, workspace, opts,undefined, undefined, useRawReqs); + await generateFlowLockInternal( + folder, + false, + workspace, + opts, + undefined, + undefined, + useRawReqs + ); } } } diff --git a/cli/src/commands/gitsync-settings/push.ts b/cli/src/commands/gitsync-settings/push.ts index f2b5bf3d72..2e7f050207 100644 --- a/cli/src/commands/gitsync-settings/push.ts +++ b/cli/src/commands/gitsync-settings/push.ts @@ -30,7 +30,7 @@ export async function pushGitSyncSettings( ) { // Validate branch configuration like sync commands try { - await validateBranchConfiguration(); + await validateBranchConfiguration({ yes: opts.yes }); } catch (error) { if (error instanceof Error && error.message.includes("overrides")) { log.error(error.message); diff --git a/cli/src/commands/hub/hub.ts b/cli/src/commands/hub/hub.ts index 79bf02ee47..314d781bba 100644 --- a/cli/src/commands/hub/hub.ts +++ b/cli/src/commands/hub/hub.ts @@ -44,18 +44,57 @@ export async function pull(opts: GlobalOptions) { "X-email": userInfo.email, }; + if (hubBaseUrl !== DEFAULT_HUB_BASE_URL) { + const hubSecret = (await wmill.getGlobal({ + key: "hub_api_secret", + })) as string | undefined; + log.info("Fetching resource types from private hub: " + hubBaseUrl); + if (hubSecret) { + log.info("Using hub API secret"); + headers["X-api-secret"] = hubSecret; + } + } + if (uid) { headers["X-uid"] = uid; } - let list = await fetch(hubBaseUrl + "/resource_types/list", { + let res1 = await fetch(hubBaseUrl + "/resource_types/list", { headers, - }).then((r) => r.json() as Promise); + }); + + if (!res1.ok) { + if (res1.status === 401) { + // 401 can only happen on a private hub + throw new Error("Unauthorized access to private hub: " + hubBaseUrl); + } else { + throw new Error( + "Couldn't fetch resource types from hub " + + hubBaseUrl + + ": " + + (await res1.text()) + ); + } + } + + let list = (await res1.json()) as HubResourceType[]; if (list && list.length === 0 && hubBaseUrl !== DEFAULT_HUB_BASE_URL) { - list = await fetch(DEFAULT_HUB_BASE_URL + "/resource_types/list", { + log.info( + "No resource types found in private hub, fetching from public hub" + ); + delete headers["X-api-secret"]; + const res2 = await fetch(DEFAULT_HUB_BASE_URL + "/resource_types/list", { headers, - }).then((r) => r.json() as Promise); + }); + + if (!res2.ok) { + throw new Error( + "Couldn't fetch resource types from public hub: " + (await res2.text()) + ); + } + + list = (await res2.json()) as HubResourceType[]; } const resourceTypes = await wmill.listResourceType({ diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 9c4171a97e..3903be6f71 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -29,7 +29,7 @@ import { parseMetadataFile, } from "../../utils/metadata.ts"; import { - LanguageWithRawReqsSupport, + LanguageWithRawReqsSupport, ScriptLanguage, inferContentTypeFromFilePath, languagesWithRawReqsSupport, @@ -114,8 +114,14 @@ export async function findResourceFile(path: string) { if (currentBranch) { // Add branch-specific candidates at the beginning (higher priority) - const branchSpecificJSON = specificItems.toBranchSpecificPath(contentBasePathJSON, currentBranch); - const branchSpecificYAML = specificItems.toBranchSpecificPath(contentBasePathYAML, currentBranch); + const branchSpecificJSON = specificItems.toBranchSpecificPath( + contentBasePathJSON, + currentBranch + ); + const branchSpecificYAML = specificItems.toBranchSpecificPath( + contentBasePathYAML, + currentBranch + ); candidates.unshift(branchSpecificJSON, branchSpecificYAML); } @@ -136,7 +142,7 @@ export async function findResourceFile(path: string) { if (validCandidates.length > 1) { throw new Error( "Found two resource files for the same resource" + - validCandidates.join(", ") + validCandidates.join(", ") ); } if (validCandidates.length < 1) { @@ -208,7 +214,8 @@ export async function handleFile( if (codebase.customBundler) { log.info(`Using custom bundler ${codebase.customBundler} for ${path}`); bundleContent = execSync( - codebase.customBundler + " " + path + codebase.customBundler + " " + path, + { maxBuffer: 1024 * 1024 * 50 } ).toString(); log.info("Custom bundler executed for " + path); } else { @@ -216,9 +223,10 @@ export async function handleFile( log.info(`Started bundling ${path} ...`); const startTime = performance.now(); + const format = codebase.format ?? "cjs"; const out = await esbuild.build({ entryPoints: [path], - format: "cjs", + format: format, bundle: true, write: false, external: codebase.external, @@ -226,7 +234,7 @@ export async function handleFile( define: codebase.define, platform: "node", packages: "bundle", - target: "node20.15.1", + target: format == "cjs" ? "node20.15.1" : "esnext", }); const endTime = performance.now(); bundleContent = out.outputFiles[0].text; @@ -266,20 +274,20 @@ export async function handleFile( let typed = opts?.skipScriptsMetadata ? undefined : ( - await parseMetadataFile( - remotePath, - opts - ? { - ...opts, - path, - workspaceRemote: workspace, - schemaOnly: codebase ? true : undefined, - } - : undefined, - globalDeps, - codebases - ) - )?.payload; + await parseMetadataFile( + remotePath, + opts + ? { + ...opts, + path, + workspaceRemote: workspace, + schemaOnly: codebase ? true : undefined, + globalDeps, + codebases + } + : undefined, + ) + )?.payload; const workspaceId = workspace.workspaceId; @@ -342,6 +350,8 @@ export async function handleFile( has_preprocessor: typed?.has_preprocessor, priority: typed?.priority, concurrency_key: typed?.concurrency_key, + debounce_key: typed?.debounce_key, + debounce_delay_s: typed?.debounce_delay_s, codebase: await codebase?.getDigest(), timeout: typed?.timeout, on_behalf_of_email: typed?.on_behalf_of_email, @@ -364,23 +374,25 @@ export async function handleFile( deepEqual(typed.schema, remote.schema) && typed.tag == remote.tag && (typed.ws_error_handler_muted ?? false) == - remote.ws_error_handler_muted && + remote.ws_error_handler_muted && typed.dedicated_worker == remote.dedicated_worker && typed.cache_ttl == remote.cache_ttl && typed.concurrency_time_window_s == - remote.concurrency_time_window_s && + remote.concurrency_time_window_s && typed.concurrent_limit == remote.concurrent_limit && Boolean(typed.restart_unless_cancelled) == - Boolean(remote.restart_unless_cancelled) && + Boolean(remote.restart_unless_cancelled) && Boolean(typed.visible_to_runner_only) == - Boolean(remote.visible_to_runner_only) && + Boolean(remote.visible_to_runner_only) && Boolean(typed.no_main_func) == Boolean(remote.no_main_func) && Boolean(typed.has_preprocessor) == - Boolean(remote.has_preprocessor) && + Boolean(remote.has_preprocessor) && typed.priority == Boolean(remote.priority) && typed.timeout == remote.timeout && //@ts-ignore typed.concurrency_key == remote["concurrency_key"] && + typed.debounce_key == remote["debounce_key"] && + typed.debounce_delay_s == remote["debounce_delay_s"] && typed.codebase == remote.codebase && typed.on_behalf_of_email == remote.on_behalf_of_email) ) { @@ -467,8 +479,7 @@ async function createScript( }); } catch (e: any) { throw Error( - `Script creation for ${body.path} with parent ${ - body.parent_hash + `Script creation for ${body.path} with parent ${body.parent_hash } was not successful: ${e.body ?? e.message} ` ); } @@ -494,8 +505,7 @@ async function createScript( }); if (req.status != 201) { throw Error( - `Script snapshot creation was not successful: ${req.status} - ${ - req.statusText + `Script snapshot creation was not successful: ${req.status} - ${req.statusText } - ${await req.text()} ` ); } @@ -507,8 +517,8 @@ export async function findContentFile(filePath: string) { const candidates = filePath.endsWith("script.json") ? exts.map((x) => filePath.replace(".script.json", x)) : filePath.endsWith("script.lock") - ? exts.map((x) => filePath.replace(".script.lock", x)) - : exts.map((x) => filePath.replace(".script.yaml", x)); + ? exts.map((x) => filePath.replace(".script.lock", x)) + : exts.map((x) => filePath.replace(".script.yaml", x)); const validCandidates = ( await Promise.all( @@ -527,7 +537,7 @@ export async function findContentFile(filePath: string) { if (validCandidates.length > 1) { throw new Error( "No content path given and more than one candidate found: " + - validCandidates.join(", ") + validCandidates.join(", ") ); } if (validCandidates.length < 1) { @@ -624,7 +634,7 @@ export const exts = [ ".nu", ".playbook.yml", ".java", - ".rb" + ".rb", // for related places search: ADD_NEW_LANG ]; @@ -727,7 +737,7 @@ async function run( if (opts.silent) { console.log(result); } else { - log.info(result); + log.info(JSON.stringify(result, null, 2)); } break; @@ -885,7 +895,10 @@ async function bootstrap( ); } -export type GlobalDeps = Map>; +export type GlobalDeps = Map< + LanguageWithRawReqsSupport, + Record +>; export async function findGlobalDeps(): Promise { var globalDeps: GlobalDeps = new Map(); @@ -895,9 +908,8 @@ export async function findGlobalDeps(): Promise { return ( !isDir && // Skip if the filename is not one of lockfile names - !(languagesWithRawReqsSupport.some( - lockfile => - p.endsWith(SEP + lockfile.rrFilename)) + !languagesWithRawReqsSupport.some((lockfile) => + p.endsWith(SEP + lockfile.rrFilename) ) ); }, els)) { @@ -906,9 +918,11 @@ export async function findGlobalDeps(): Promise { // Iterate over available languages to find which lockfile languagesWithRawReqsSupport.map((lock) => { - if (entry.path.endsWith(lock.rrFilename)){ + if (entry.path.endsWith(lock.rrFilename)) { const current = globalDeps.get(lock) ?? {}; - current[entry.path.substring(0, entry.path.length - lock.rrFilename.length)] = content; + current[ + entry.path.substring(0, entry.path.length - lock.rrFilename.length) + ] = content; globalDeps.set(lock, current); } }); @@ -924,7 +938,7 @@ async function generateMetadata( scriptPath: string | undefined ) { log.info( - "This command only works for workspace scripts, for flows inline scripts use `wmill flow generate - locks`" + "This command only works for workspace scripts, for flows inline scripts use `wmill flow generate-locks`" ); if (scriptPath == "") { scriptPath = undefined; diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 4ab1af2252..429fbd6ebc 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -4,7 +4,6 @@ import { colors, Command, Confirm, - Select, ensureDir, minimatch, JSZip, @@ -39,7 +38,6 @@ import { handleFile } from "../script/script.ts"; import { deepEqual, isFileResource } from "../../utils/utils.ts"; import { SyncOptions, - readConfigFile, getEffectiveSettings, validateBranchConfiguration, mergeConfigWithConfigFile, @@ -51,7 +49,6 @@ import { getBranchSpecificPath, fromBranchSpecificPath, isCurrentBranchFile, - toBranchSpecificPath, isBranchSpecificFile, } from "../../core/specific_items.ts"; import { getCurrentGitBranch } from "../../utils/git.ts"; @@ -65,7 +62,10 @@ import { } from "../../utils/metadata.ts"; import { OpenFlow } from "../../../gen/types.gen.ts"; import { pushResource } from "../resource/resource.ts"; -import { newPathAssigner, PathAssigner } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; +import { + newPathAssigner, + PathAssigner, +} from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; // Merge CLI options with effective settings, preserving CLI flags as overrides @@ -158,7 +158,9 @@ async function addCodebaseDigestIfRelevant( try { parsed = yamlParseContent(path, content); } catch (error) { - log.error(`Failed to parse YAML content for codebase digest at path: ${path}`); + log.error( + `Failed to parse YAML content for codebase digest at path: ${path}` + ); throw error; } if (parsed && typeof parsed == "object") { @@ -258,7 +260,10 @@ export interface InlineScript { content: string; } -export function extractInlineScriptsForApps(rec: any, pathAssigner: PathAssigner): InlineScript[] { +export function extractInlineScriptsForApps( + rec: any, + pathAssigner: PathAssigner +): InlineScript[] { if (!rec) { return []; } @@ -349,10 +354,12 @@ function ZipFSElement( flow.value.modules, {}, SEP, - defaultTs, + defaultTs ); } catch (error) { - log.error(`Failed to extract inline scripts for flow at path: ${p}`); + log.error( + `Failed to extract inline scripts for flow at path: ${p}` + ); throw error; } for (const s of inlineScripts) { @@ -386,9 +393,14 @@ function ZipFSElement( } let inlineScripts; try { - inlineScripts = extractInlineScriptsForApps(app?.["value"], newPathAssigner(defaultTs)); + inlineScripts = extractInlineScriptsForApps( + app?.["value"], + newPathAssigner(defaultTs) + ); } catch (error) { - log.error(`Failed to extract inline scripts for app at path: ${p}`); + log.error( + `Failed to extract inline scripts for app at path: ${p}` + ); throw error; } for (const s of inlineScripts) { @@ -913,13 +925,17 @@ async function compareDynFSElement( try { parsedV = JSON.parse(v); } catch (error) { - log.error(`Failed to parse new JSON content for comparison at path: ${k}`); + log.error( + `Failed to parse new JSON content for comparison at path: ${k}` + ); throw error; } try { parsedM2 = JSON.parse(m2[k]); } catch (error) { - log.error(`Failed to parse existing JSON content for comparison at path: ${k}`); + log.error( + `Failed to parse existing JSON content for comparison at path: ${k}` + ); throw error; } if (deepEqual(parsedV, parsedM2)) { @@ -932,11 +948,11 @@ async function compareDynFSElement( continue; } if (!ignoreCodebaseChanges) { - if (before.codebase != undefined) { + if (before?.codebase != undefined) { delete before.codebase; m2[k] = yamlStringify(before, yamlOptions); } - if (after.codebase != undefined) { + if (after?.codebase != undefined) { if (before.codebase != after.codebase) { codebaseChanges[k] = after.codebase; } @@ -1241,13 +1257,12 @@ export async function pull( opts: GlobalOptions & SyncOptions & { repository?: string; promotion?: string } ) { - const originalCliOpts = { ...opts }; opts = await mergeConfigWithConfigFile(opts); // Validate branch configuration early try { - await validateBranchConfiguration(false, opts.yes); + await validateBranchConfiguration(opts); } catch (error) { if (error instanceof Error && error.message.includes("overrides")) { log.error(error.message); @@ -1345,7 +1360,10 @@ export async function pull( ...(specificItems && isSpecificItem(change.path, specificItems) ? { branch_specific: true, - branch_specific_path: getBranchSpecificPath(change.path, specificItems) + branch_specific_path: getBranchSpecificPath( + change.path, + specificItems + ), } : {}), })), @@ -1380,7 +1398,10 @@ export async function pull( // Determine if this file should be written to a branch-specific path let targetPath = change.path; if (specificItems && isSpecificItem(change.path, specificItems)) { - const branchSpecificPath = getBranchSpecificPath(change.path, specificItems); + const branchSpecificPath = getBranchSpecificPath( + change.path, + specificItems + ); if (branchSpecificPath) { targetPath = branchSpecificPath; } @@ -1430,12 +1451,24 @@ export async function pull( } } if (exts.some((e) => change.path.endsWith(e))) { - log.info(`Editing script content of ${targetPath}${targetPath !== change.path ? colors.gray(` (branch-specific override for ${change.path})`) : ""}`); + log.info( + `Editing script content of ${targetPath}${ + targetPath !== change.path + ? colors.gray(` (branch-specific override for ${change.path})`) + : "" + }` + ); } else if ( change.path.endsWith(".yaml") || change.path.endsWith(".json") ) { - log.info(`Editing ${getTypeStrFromPath(change.path)} ${targetPath}${targetPath !== change.path ? colors.gray(` (branch-specific override for ${change.path})`) : ""}`); + log.info( + `Editing ${getTypeStrFromPath(change.path)} ${targetPath}${ + targetPath !== change.path + ? colors.gray(` (branch-specific override for ${change.path})`) + : "" + }` + ); } await Deno.writeTextFile(target, change.after); @@ -1447,10 +1480,22 @@ export async function pull( await ensureDir(path.dirname(target)); if (opts.stateful) { await ensureDir(path.dirname(stateTarget)); - log.info(`Adding ${getTypeStrFromPath(change.path)} ${targetPath}${targetPath !== change.path ? colors.gray(` (branch-specific override for ${change.path})`) : ""}`); + log.info( + `Adding ${getTypeStrFromPath(change.path)} ${targetPath}${ + targetPath !== change.path + ? colors.gray(` (branch-specific override for ${change.path})`) + : "" + }` + ); } await Deno.writeTextFile(target, change.content); - log.info(`Writing ${getTypeStrFromPath(change.path)} ${targetPath}${targetPath !== change.path ? colors.gray(` (branch-specific override for ${change.path})`) : ""}`); + log.info( + `Writing ${getTypeStrFromPath(change.path)} ${targetPath}${ + targetPath !== change.path + ? colors.gray(` (branch-specific override for ${change.path})`) + : "" + }` + ); if (opts.stateful) { await Deno.copyFile(target, stateTarget); } @@ -1528,7 +1573,10 @@ export async function pull( ...(specificItems && isSpecificItem(change.path, specificItems) ? { branch_specific: true, - branch_specific_path: getBranchSpecificPath(change.path, specificItems) + branch_specific_path: getBranchSpecificPath( + change.path, + specificItems + ), } : {}), })), @@ -1560,7 +1608,10 @@ function prettyChanges(changes: Change[], specificItems?: SpecificItemsConfig) { // Check if this will be written as a branch-specific file if (specificItems && isSpecificItem(change.path, specificItems)) { - const branchSpecificPath = getBranchSpecificPath(change.path, specificItems); + const branchSpecificPath = getBranchSpecificPath( + change.path, + specificItems + ); if (branchSpecificPath) { displayPath = branchSpecificPath; branchNote = " (branch-specific)"; @@ -1569,22 +1620,48 @@ function prettyChanges(changes: Change[], specificItems?: SpecificItemsConfig) { if (change.name === "added") { log.info( - colors.green(`+ ${getTypeStrFromPath(change.path)} ` + displayPath + colors.gray(branchNote)) + colors.green( + `+ ${getTypeStrFromPath(change.path)} ` + + displayPath + + colors.gray(branchNote) + ) ); } else if (change.name === "deleted") { log.info( - colors.red(`- ${getTypeStrFromPath(change.path)} ` + displayPath + colors.gray(branchNote)) + colors.red( + `- ${getTypeStrFromPath(change.path)} ` + + displayPath + + colors.gray(branchNote) + ) ); } else if (change.name === "edited") { log.info( colors.yellow( `~ ${getTypeStrFromPath(change.path)} ` + - displayPath + colors.gray(branchNote) + + displayPath + + colors.gray(branchNote) + (change.codebase ? ` (codebase changed)` : "") ) ); if (change.before != change.after) { - showDiff(change.before, change.after); + if (change.path.endsWith(".yaml")) { + try { + showDiff( + yamlStringify( + yamlParseContent(change.path, change.before), + yamlOptions + ), + yamlStringify( + yamlParseContent(change.path, change.after), + yamlOptions + ) + ); + } catch { + showDiff(change.before, change.after); + } + } else { + showDiff(change.before, change.after); + } } } } @@ -1627,7 +1704,7 @@ export async function push( // Validate branch configuration early try { - await validateBranchConfiguration(false, opts.yes); + await validateBranchConfiguration(opts); } catch (error) { if (error instanceof Error && error.message.includes("overrides")) { log.error(error.message); @@ -1794,7 +1871,10 @@ export async function push( ...(specificItems && isSpecificItem(change.path, specificItems) ? { branch_specific: true, - branch_specific_path: getBranchSpecificPath(change.path, specificItems) + branch_specific_path: getBranchSpecificPath( + change.path, + specificItems + ), } : {}), })), @@ -1938,7 +2018,10 @@ export async function push( const currentBranch = getCurrentGitBranch(); if (currentBranch && isBranchSpecificFile(resourceFilePath)) { - serverPath = fromBranchSpecificPath(resourceFilePath, currentBranch); + serverPath = fromBranchSpecificPath( + resourceFilePath, + currentBranch + ); } await pushResource( @@ -1960,7 +2043,10 @@ export async function push( // Check if this is a branch-specific item and get the original branch-specific path let originalBranchSpecificPath: string | undefined; if (specificItems && isSpecificItem(change.path, specificItems)) { - originalBranchSpecificPath = getBranchSpecificPath(change.path, specificItems); + originalBranchSpecificPath = getBranchSpecificPath( + change.path, + specificItems + ); } await pushObj( @@ -2010,7 +2096,10 @@ export async function push( // For branch-specific items, we read from branch-specific files but push to base server paths let localFilePath = change.path; if (specificItems && isSpecificItem(change.path, specificItems)) { - const branchSpecificPath = getBranchSpecificPath(change.path, specificItems); + const branchSpecificPath = getBranchSpecificPath( + change.path, + specificItems + ); if (branchSpecificPath) { localFilePath = branchSpecificPath; } @@ -2024,7 +2113,7 @@ export async function push( opts.plainSecrets ?? false, [], opts.message, - localFilePath // Pass the actual local file path + localFilePath // Pass the actual local file path ); if (stateTarget) { @@ -2045,14 +2134,10 @@ export async function push( const target = change.path.replaceAll(SEP, "/"); switch (typ) { case "script": { - const script = await wmill.getScriptByPath({ + await wmill.archiveScriptByPath({ workspace: workspaceId, path: removeExtensionToPath(target), }); - await wmill.archiveScriptByHash({ - workspace: workspaceId, - hash: script.hash, - }); break; } case "folder": @@ -2216,7 +2301,10 @@ export async function push( ...(specificItems && isSpecificItem(change.path, specificItems) ? { branch_specific: true, - branch_specific_path: getBranchSpecificPath(change.path, specificItems) + branch_specific_path: getBranchSpecificPath( + change.path, + specificItems + ), } : {}), })), @@ -2280,6 +2368,7 @@ const command = new Command() .option("--include-groups", "Include syncing groups") .option("--include-settings", "Include syncing workspace settings") .option("--include-key", "Include workspace encryption key") + .option("--skip-branch-validation", "Skip git branch validation and prompts") .option("--json-output", "Output results in JSON format") .option( "-i --includes ", @@ -2327,6 +2416,7 @@ const command = new Command() .option("--include-groups", "Include syncing groups") .option("--include-settings", "Include syncing workspace settings") .option("--include-key", "Include workspace encryption key") + .option("--skip-branch-validation", "Skip git branch validation and prompts") .option("--json-output", "Output results in JSON format") .option( "-i --includes ", diff --git a/cli/src/commands/workspace/fork.ts b/cli/src/commands/workspace/fork.ts index 04759a09d3..1a0d3a45b7 100644 --- a/cli/src/commands/workspace/fork.ts +++ b/cli/src/commands/workspace/fork.ts @@ -1,9 +1,7 @@ // deno-lint-ignore-file no-explicit-any import { GlobalOptions } from "../../types.ts"; -import { colors, Command, Input, log, setClient } from "../../../deps.ts"; -import { requireLogin } from "../../core/auth.ts"; -import { add, addWorkspace, allWorkspaces, getActiveWorkspace, list, removeWorkspace } from "./workspace.ts"; -import { loginInteractive, tryGetLoginInfo } from "../../core/login.ts"; +import { colors, Input, log, setClient } from "../../../deps.ts"; +import { addWorkspace, allWorkspaces, list, removeWorkspace } from "./workspace.ts"; import * as wmill from "../../../gen/services.gen.ts"; import { getCurrentGitBranch, getOriginalBranchForWorkspaceForks, isGitRepository } from "../../utils/git.ts"; import { WM_FORK_PREFIX } from "../../main.ts"; @@ -107,12 +105,11 @@ async function createWorkspaceFork( try { // TODO: Update to createWorkspaceFork after regenerating client from new OpenAPI spec const result = await wmill.createWorkspaceFork({ + workspace: workspace.workspaceId, requestBody: { id: trueWorkspaceId, name: opts.createWorkspaceName ?? trueWorkspaceId, - username: undefined, // Let the server handle username color: undefined, - parent_workspace_id: workspace.workspaceId, }, }); diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index ebe871959f..5a56f97435 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -1,5 +1,9 @@ import { log, yamlParseFile, Confirm, yamlStringify } from "../../deps.ts"; -import { getCurrentGitBranch, getOriginalBranchForWorkspaceForks, isGitRepository } from "../utils/git.ts"; +import { + getCurrentGitBranch, + getOriginalBranchForWorkspaceForks, + isGitRepository, +} from "../utils/git.ts"; import { join, dirname, resolve, relative } from "node:path"; import { existsSync } from "node:fs"; import { execSync } from "node:child_process"; @@ -32,6 +36,7 @@ export interface SyncOptions { includeGroups?: boolean; includeSettings?: boolean; includeKey?: boolean; + skipBranchValidation?: boolean; message?: string; includes?: string[]; extraIncludes?: string[]; @@ -94,13 +99,14 @@ export interface Codebase { external?: string[]; define?: { [key: string]: string }; inject?: string[]; + format?: "cjs" | "esm"; } function getGitRepoRoot(): string | null { try { const result = execSync("git rev-parse --show-toplevel", { encoding: "utf8", - stdio: "pipe" + stdio: "pipe", }); return result.trim(); } catch (error) { @@ -182,34 +188,45 @@ export async function readConfigFile(): Promise { const migrationMessages: string[] = []; // Handle obsolete overrides format - if (conf && 'overrides' in conf) { + if (conf && "overrides" in conf) { const overrides = conf.overrides as any; - const hasSettings = overrides && typeof overrides === 'object' && Object.keys(overrides).length > 0; + const hasSettings = + overrides && + typeof overrides === "object" && + Object.keys(overrides).length > 0; if (hasSettings) { throw new Error( "❌ The 'overrides' field is no longer supported.\n" + - " The configuration system now uses Git branch-based configuration only.\n" + - " Please delete your wmill.yaml and run 'wmill init' to recreate it with the new format." + " The configuration system now uses Git branch-based configuration only.\n" + + " Please delete your wmill.yaml and run 'wmill init' to recreate it with the new format." ); } else { // Remove empty overrides delete conf.overrides; needsConfigWrite = true; - migrationMessages.push("ℹ️ Removing empty 'overrides: {}' from wmill.yaml (migrated to gitBranches format)"); + migrationMessages.push( + "ℹ️ Removing empty 'overrides: {}' from wmill.yaml (migrated to gitBranches format)" + ); } } // Handle git_branches to gitBranches migration - if (conf && 'git_branches' in conf) { + if (conf && "git_branches" in conf) { if (!conf.gitBranches) { // Deep copy git_branches to gitBranches (even if empty) conf.gitBranches = JSON.parse(JSON.stringify(conf.git_branches)); needsConfigWrite = true; - migrationMessages.push("⚠️ Migrating 'git_branches' to 'gitBranches' (camelCase). The snake_case format is deprecated."); - migrationMessages.push("✅ Successfully migrated 'git_branches' to 'gitBranches' in wmill.yaml"); + migrationMessages.push( + "⚠️ Migrating 'git_branches' to 'gitBranches' (camelCase). The snake_case format is deprecated." + ); + migrationMessages.push( + "✅ Successfully migrated 'git_branches' to 'gitBranches' in wmill.yaml" + ); } else { - migrationMessages.push("⚠️ Both 'git_branches' and 'gitBranches' found in wmill.yaml. Using 'gitBranches' and ignoring 'git_branches'."); + migrationMessages.push( + "⚠️ Both 'git_branches' and 'gitBranches' found in wmill.yaml. Using 'gitBranches' and ignoring 'git_branches'." + ); } // Always remove the old field from config object (both file and memory) delete conf.git_branches; @@ -220,20 +237,24 @@ export async function readConfigFile(): Promise { try { await Deno.writeTextFile(wmillYamlPath, yamlStringify(conf)); // Log all migration messages after successful write - migrationMessages.forEach(msg => { - if (msg.startsWith('⚠️')) { + migrationMessages.forEach((msg) => { + if (msg.startsWith("⚠️")) { log.warn(msg); } else { log.info(msg); } }); } catch (error) { - log.warn(`Could not update wmill.yaml to apply migrations: ${error instanceof Error ? error.message : error}`); + log.warn( + `Could not update wmill.yaml to apply migrations: ${ + error instanceof Error ? error.message : error + }` + ); } } else if (migrationMessages.length > 0) { // Log messages for non-write cases (like "both found") - migrationMessages.forEach(msg => { - if (msg.startsWith('⚠️')) { + migrationMessages.forEach((msg) => { + if (msg.startsWith("⚠️")) { log.warn(msg); } else { log.info(msg); @@ -248,38 +269,66 @@ export async function readConfigFile(): Promise { } return typeof conf == "object" ? conf : ({} as SyncOptions); } catch (e) { - if (e instanceof Error && (e.message.includes("overrides") || e.message.includes("Obsolete configuration format"))) { + if ( + e instanceof Error && + (e.message.includes("overrides") || + e.message.includes("Obsolete configuration format")) + ) { throw e; // Re-throw the specific obsolete format error } // Since we already found the file path, this is likely a parsing or access error if (e instanceof Error && e.message.includes("Error parsing yaml")) { - const yamlError = e.cause instanceof Error ? e.cause.message : String(e.cause); + const yamlError = + e.cause instanceof Error ? e.cause.message : String(e.cause); throw new Error( "❌ YAML syntax error in wmill.yaml:\n" + - " " + yamlError + "\n" + - " Please fix the YAML syntax in wmill.yaml or delete the file to start fresh." + " " + + yamlError + + "\n" + + " Please fix the YAML syntax in wmill.yaml or delete the file to start fresh." ); } else { // File exists but has other issues (permissions, etc.) throw new Error( "❌ Failed to read wmill.yaml:\n" + - " " + (e instanceof Error ? e.message : String(e)) + "\n" + - " Please check file permissions or fix the syntax." + " " + + (e instanceof Error ? e.message : String(e)) + + "\n" + + " Please check file permissions or fix the syntax." ); } } } // Default sync options - shared across the codebase to prevent duplication -export const DEFAULT_SYNC_OPTIONS: Readonly>> = { - defaultTs: 'bun', - includes: ['f/**'], +export const DEFAULT_SYNC_OPTIONS: Readonly< + Required< + Pick< + SyncOptions, + | "defaultTs" + | "includes" + | "excludes" + | "codebases" + | "skipVariables" + | "skipResources" + | "skipResourceTypes" + | "skipSecrets" + | "includeSchedules" + | "includeTriggers" + | "skipScripts" + | "skipFlows" + | "skipApps" + | "skipFolders" + | "includeUsers" + | "includeGroups" + | "includeSettings" + | "includeKey" + > + > +> = { + defaultTs: "bun", + includes: ["f/**"], excludes: [], codebases: [], skipVariables: false, @@ -295,7 +344,7 @@ export const DEFAULT_SYNC_OPTIONS: Readonly( @@ -306,8 +355,10 @@ export async function mergeConfigWithConfigFile( } // Validate branch configuration early in the process -export async function validateBranchConfiguration(skipValidation?: boolean, autoAccept?: boolean): Promise { - if (skipValidation || !isGitRepository()) { +export async function validateBranchConfiguration( + opts: Pick +): Promise { + if (opts.skipBranchValidation || !isGitRepository()) { return; } @@ -320,7 +371,9 @@ export async function validateBranchConfiguration(skipValidation?: boolean, auto let currentBranch: string | null; if (originalBranchIfForked) { - log.info(`Workspace fork detected from branch name \`${rawBranch}\`. Validating branch configuration using original branch \`${originalBranchIfForked}\``); + log.info( + `Workspace fork detected from branch name \`${rawBranch}\`. Validating branch configuration using original branch \`${originalBranchIfForked}\`` + ); currentBranch = originalBranchIfForked; } else { currentBranch = rawBranch; @@ -330,8 +383,8 @@ export async function validateBranchConfiguration(skipValidation?: boolean, auto if (!gitBranches || Object.keys(gitBranches).length === 0) { log.warn( "⚠️ WARNING: In a Git repository, the 'gitBranches' section is recommended in wmill.yaml.\n" + - " Consider adding a gitBranches section with configuration for your Git branches.\n" + - " Run 'wmill init' to recreate the configuration file with proper branch setup." + " Consider adding a gitBranches section with configuration for your Git branches.\n" + + " Run 'wmill init' to recreate the configuration file with proper branch setup." ); return; } @@ -340,24 +393,35 @@ export async function validateBranchConfiguration(skipValidation?: boolean, auto if (currentBranch && !gitBranches[currentBranch]) { // In interactive mode, offer to create the branch if (Deno.stdin.isTerminal()) { - const availableBranches = Object.keys(gitBranches).join(', '); + const availableBranches = Object.keys(gitBranches).join(", "); log.info( `Current Git branch '${currentBranch}' is not defined in the gitBranches configuration.\n` + - `Available branches: ${availableBranches}` + `Available branches: ${availableBranches}` ); - const shouldCreate = autoAccept || await Confirm.prompt({ - message: `Create empty branch configuration for '${currentBranch}'?`, - default: true, - }); + const shouldCreate = + opts.yes || + (await Confirm.prompt({ + message: `Create empty branch configuration for '${currentBranch}'?`, + default: true, + })); if (shouldCreate) { // Warn if branch name contains filesystem-unsafe characters if (/[\/\\:*?"<>|.]/.test(currentBranch)) { - const sanitizedBranchName = currentBranch.replace(/[\/\\:*?"<>|.]/g, '_'); - log.warn(`⚠️ WARNING: Branch name "${currentBranch}" contains filesystem-unsafe characters (/ \\ : * ? " < > | .).`); - log.warn(` Branch-specific files will be saved with sanitized name: "${sanitizedBranchName}"`); - log.warn(` Example: "file.variable.yaml" → "file.${sanitizedBranchName}.variable.yaml"`); + const sanitizedBranchName = currentBranch.replace( + /[\/\\:*?"<>|.]/g, + "_" + ); + log.warn( + `⚠️ WARNING: Branch name "${currentBranch}" contains filesystem-unsafe characters (/ \\ : * ? " < > | .).` + ); + log.warn( + ` Branch-specific files will be saved with sanitized name: "${sanitizedBranchName}"` + ); + log.warn( + ` Example: "file.variable.yaml" → "file.${sanitizedBranchName}.variable.yaml"` + ); } // Read current config, add branch, and write it back @@ -370,23 +434,34 @@ export async function validateBranchConfiguration(skipValidation?: boolean, auto await Deno.writeTextFile("wmill.yaml", yamlStringify(currentConfig)); - log.info(`✅ Created empty branch configuration for '${currentBranch}'`); + log.info( + `✅ Created empty branch configuration for '${currentBranch}'` + ); } else { - log.warn("⚠️ WARNING: Branch creation cancelled. You can manually add the branch to wmill.yaml or use 'wmill gitsync-settings pull' to pull configuration from an existing windmill workspace git-sync configuration."); + log.warn( + "⚠️ WARNING: Branch creation cancelled. You can manually add the branch to wmill.yaml or use 'wmill gitsync-settings pull' to pull configuration from an existing windmill workspace git-sync configuration." + ); return; } } else { // Warn about filesystem-unsafe characters in branch name if (/[\/\\:*?"<>|.]/.test(currentBranch)) { - const sanitizedBranchName = currentBranch.replace(/[\/\\:*?"<>|.]/g, '_'); - log.warn(`⚠️ WARNING: Branch name "${currentBranch}" contains filesystem-unsafe characters (/ \\ : * ? " < > | .).`); - log.warn(` Branch-specific files will use sanitized name: "${sanitizedBranchName}"`); + const sanitizedBranchName = currentBranch.replace( + /[\/\\:*?"<>|.]/g, + "_" + ); + log.warn( + `⚠️ WARNING: Branch name "${currentBranch}" contains filesystem-unsafe characters (/ \\ : * ? " < > | .).` + ); + log.warn( + ` Branch-specific files will use sanitized name: "${sanitizedBranchName}"` + ); } - + log.warn( `⚠️ WARNING: Current Git branch '${currentBranch}' is not defined in the gitBranches configuration.\n` + - ` Consider adding configuration for branch '${currentBranch}' in the gitBranches section of wmill.yaml.\n` + - ` Available branches: ${Object.keys(gitBranches).join(', ')}` + ` Consider adding configuration for branch '${currentBranch}' in the gitBranches section of wmill.yaml.\n` + + ` Available branches: ${Object.keys(gitBranches).join(", ")}` ); return; } @@ -394,7 +469,12 @@ export async function validateBranchConfiguration(skipValidation?: boolean, auto } // Get effective settings by merging top-level settings with branch-specific overrides -export async function getEffectiveSettings(config: SyncOptions, promotion?: string, skipBranchValidation?: boolean, suppressLogs?: boolean): Promise { +export async function getEffectiveSettings( + config: SyncOptions, + promotion?: string, + skipBranchValidation?: boolean, + suppressLogs?: boolean +): Promise { // Start with top-level settings from config const { gitBranches, ...topLevelSettings } = config; const effective = { ...topLevelSettings }; @@ -406,10 +486,12 @@ export async function getEffectiveSettings(config: SyncOptions, promotion?: stri let currentBranch: string | null; if (originalBranchIfForked) { - log.info(`Using overrides from original branch \`${originalBranchIfForked}\``); + log.info( + `Using overrides from original branch \`${originalBranchIfForked}\`` + ); currentBranch = originalBranchIfForked; } else { - currentBranch = branch + currentBranch = branch; } // If promotion is specified, use that branch's promotionOverrides or overrides @@ -425,21 +507,36 @@ export async function getEffectiveSettings(config: SyncOptions, promotion?: stri } else if (targetBranch.overrides) { Object.assign(effective, targetBranch.overrides); if (!suppressLogs) { - log.info(`Applied settings from branch: ${promotion} (no promotionOverrides found)`); + log.info( + `Applied settings from branch: ${promotion} (no promotionOverrides found)` + ); } } else { - log.debug(`No promotion or regular overrides found for branch '${promotion}', using top-level settings`); + log.debug( + `No promotion or regular overrides found for branch '${promotion}', using top-level settings` + ); } } // Otherwise use current branch overrides (existing behavior) - else if (currentBranch && gitBranches && gitBranches[currentBranch] && gitBranches[currentBranch].overrides) { + else if ( + currentBranch && + gitBranches && + gitBranches[currentBranch] && + gitBranches[currentBranch].overrides + ) { Object.assign(effective, gitBranches[currentBranch].overrides); if (!suppressLogs) { - const extraLog = originalBranchIfForked ? ` (because it is the origin of the workspace fork branch \`${branch}\`)` : ""; - log.info(`Applied settings for Git branch: ${currentBranch}${extraLog}`); + const extraLog = originalBranchIfForked + ? ` (because it is the origin of the workspace fork branch \`${branch}\`)` + : ""; + log.info( + `Applied settings for Git branch: ${currentBranch}${extraLog}` + ); } } else if (currentBranch) { - log.debug(`No branch-specific overrides found for '${currentBranch}', using top-level settings`); + log.debug( + `No branch-specific overrides found for '${currentBranch}', using top-level settings` + ); } } else { log.debug("Not in a Git repository, using top-level settings"); diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index 00b69128fd..c31a4d6338 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -78,7 +78,7 @@ export async function pushWorkspaceSettings( error_handler: remoteSettings.error_handler, error_handler_extra_args: remoteSettings.error_handler_extra_args, error_handler_muted_on_cancel: - remoteSettings.error_handler_muted_on_cancel, + remoteSettings.error_handler_muted_on_cancel ?? false, ai_config: remoteSettings.ai_config, large_file_storage: remoteSettings.large_file_storage, git_sync: remoteSettings.git_sync, @@ -166,8 +166,8 @@ export async function pushWorkspaceSettings( localSettings.error_handler_extra_args, settings.error_handler_extra_args ) || - localSettings.error_handler_muted_on_cancel != - settings.error_handler_muted_on_cancel + (localSettings.error_handler_muted_on_cancel ?? false) != + (settings.error_handler_muted_on_cancel ?? false) ) { log.debug(`Updating error handler...`); await wmill.editErrorHandler({ @@ -176,7 +176,7 @@ export async function pushWorkspaceSettings( error_handler: localSettings.error_handler, error_handler_extra_args: localSettings.error_handler_extra_args, error_handler_muted_on_cancel: - localSettings.error_handler_muted_on_cancel, + localSettings.error_handler_muted_on_cancel ?? false, }, }); } diff --git a/cli/src/guidance/flow_guidance.ts b/cli/src/guidance/flow_guidance.ts index 4766a7725b..4b10e946b3 100644 --- a/cli/src/guidance/flow_guidance.ts +++ b/cli/src/guidance/flow_guidance.ts @@ -29,6 +29,8 @@ value: concurrent_limit: 0 # Limit concurrent executions concurrency_key: "string" # Custom concurrency grouping concurrency_time_window_s: 0 + custom_debounce_key: "key" + debounce_delay_s: 0 skip_expr: "javascript_expression" # Skip workflow condition cache_ttl: 0 # Cache results duration priority: 0 # Execution priority @@ -59,6 +61,8 @@ value: concurrent_limit: 0 concurrency_time_window_s: 0 custom_concurrency_key: "key" + custom_debounce_key: "key" + debounce_delay_s: 0 is_trigger: false assets: [] \`\`\` @@ -427,4 +431,4 @@ schema: \`\`\` When generating OpenFlow YAML, ensure proper indentation, valid YAML syntax, and logical step dependencies. Always include meaningful summaries and proper input transforms to connect workflow steps. -`; \ No newline at end of file +`; diff --git a/cli/src/main.ts b/cli/src/main.ts index d594616521..c0ed2b651b 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { // } // }); -export const VERSION = "1.542.1"; +export const VERSION = "1.573.3"; export const WM_FORK_PREFIX = "wm-fork"; @@ -187,7 +187,7 @@ async function main() { log.setup({ handlers: { console: new log.ConsoleHandler(LOG_LEVEL, { - formatter: ({ msg }) => `${msg}`, + formatter: ({ msg }) => msg, useColors: isWin ? false : true, }), }, diff --git a/cli/src/types.ts b/cli/src/types.ts index ec3600b0a8..a7b821882a 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -301,18 +301,30 @@ export function getTypeStrFromPath( } export function removeType(str: string, type: string) { + // Normalize path for cross-platform compatibility and convert to forward slashes for API consistency + const normalizedStr = path.normalize(str).replaceAll(SEP, "/"); + if ( - !str.endsWith("." + type + ".yaml") && - !str.endsWith("." + type + ".json") + !normalizedStr.endsWith("." + type + ".yaml") && + !normalizedStr.endsWith("." + type + ".json") ) { throw new Error(str + " does not end with ." + type + ".(yaml|json)"); } - return str.slice(0, str.length - type.length - 6); + return normalizedStr.slice(0, normalizedStr.length - type.length - 6); } export function removePathPrefix(str: string, prefix: string) { - if (!str.startsWith(prefix + "/")) { + // Normalize paths for cross-platform compatibility and convert to forward slashes for API consistency + const normalizedStr = path.normalize(str).replaceAll(SEP, "/"); + const normalizedPrefix = path.normalize(prefix).replaceAll(SEP, "/"); + + // Handle exact match case + if (normalizedStr === normalizedPrefix) { + return ""; + } + + if (!normalizedStr.startsWith(normalizedPrefix + "/")) { throw new Error(str + " does not start with " + prefix); } - return str.slice(prefix.length + 1); + return normalizedStr.slice(normalizedPrefix.length + 1); } diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index d55cbfc930..00e2206933 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -20,8 +20,17 @@ import { } from "./script_common.ts"; import { inferContentTypeFromFilePath } from "./script_common.ts"; import { GlobalDeps, exts, findGlobalDeps } from "../commands/script/script.ts"; -import { FSFSElement, findCodebase, yamlOptions } from "../commands/sync/sync.ts"; -import { generateHash, readInlinePathSync, getHeaders } from "./utils.ts"; +import { + FSFSElement, + findCodebase, + yamlOptions, +} from "../commands/sync/sync.ts"; +import { + generateHash, + readInlinePathSync, + getHeaders, + writeIfChanged, +} from "./utils.ts"; import { SyncCodebase } from "./codebase.ts"; import { FlowFile } from "../commands/flow/flow.ts"; import { replaceInlineScripts } from "../../windmill-utils-internal/src/inline-scripts/replacer.ts"; @@ -77,7 +86,6 @@ async function generateFlowHash( [, reqs] = Object.entries(rawReqs).find(([lang2, _]) => lang == lang2) ?? []; } - // Embed lock into hash hashes[f.path] = await generateHash( (await f.getContentText()) + (reqs ?? "") @@ -175,8 +183,8 @@ export async function generateFlowLockInternal( folder + SEP!, SEP, changedScripts, - (path: string, newPath: string) => Deno.renameSync(path, newPath), - (path: string) => Deno.removeSync(path) + // (path: string, newPath: string) => Deno.renameSync(path, newPath), + // (path: string) => Deno.removeSync(path) ); //removeChangedLocks @@ -191,26 +199,21 @@ export async function generateFlowLockInternal( flowValue.value.modules, {}, SEP, - opts.defaultTs, + opts.defaultTs ); - inlineScripts - .filter((s) => s.path.endsWith(".lock")) - .forEach((s) => { - Deno.writeTextFileSync( - Deno.cwd() + SEP + folder + SEP + s.path, - s.content - ); - }); + inlineScripts.forEach((s) => { + writeIfChanged(Deno.cwd() + SEP + folder + SEP + s.path, s.content); + }); // Overwrite `flow.yaml` with the new lockfile references - await Deno.writeTextFile( + writeIfChanged( Deno.cwd() + SEP + folder + SEP + "flow.yaml", yamlStringify(flowValue as Record) ); } hashes = await generateFlowHash(rawReqs, folder, opts.defaultTs); - + await clearGlobalLock(folder); for (const [path, hash] of Object.entries(hashes)) { await updateMetadataGlobalLock(folder, hash, path); } @@ -243,29 +246,22 @@ export async function generateScriptMetadataInternal( const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs); + + const metadataWithType = await parseMetadataFile( + remotePath, + undefined, + ); + + // read script content + const scriptContent = await Deno.readTextFile(scriptPath); + const metadataContent = await Deno.readTextFile(metadataWithType.path); + const rrLang = languagesWithRawReqsSupport.find( (l) => language == l.language ); const rawReqs = findClosestRawReqs(rrLang, scriptPath, globalDeps); - if (rawReqs && rrLang) { - log.info( - (await blueColor())( - `Found raw requirements (${rrLang.rrFilename}) for ${scriptPath}, using it` - ) - ); - } - const metadataWithType = await parseMetadataFile( - remotePath, - undefined, - globalDeps, - codebases - ); - - // read script content - const scriptContent = await Deno.readTextFile(scriptPath); - const metadataContent = await Deno.readTextFile(metadataWithType.path); let hash = await generateScriptHash(rawReqs, scriptContent, metadataContent); @@ -382,6 +378,10 @@ async function updateScriptLock( ) { return; } + + if (rawDeps) { + log.info(`Generating script lock for ${remotePath} with raw deps`); + } // generate the script lock running a dependency job in Windmill and update it inplace // TODO: update this once the client is released const extraHeaders = getHeaders(); @@ -432,7 +432,9 @@ async function updateScriptLock( if (await Deno.stat(lockPath)) { await Deno.remove(lockPath); } - } catch {} + } catch (e) { + log.info(colors.yellow(`Error removing lock file ${lockPath}: ${e}`)); + } metadataContent.lock = ""; } } catch (e) { @@ -516,7 +518,9 @@ export async function updateFlow( } catch (e) { try { responseText = await rawResponse.text(); - } catch {} + } catch { + responseText = ""; + } throw new Error( `Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}` ); @@ -538,16 +542,24 @@ export async function inferSchema( }> { let inferedSchema: any; if (language === "python3") { - const { parse_python } = await import("../../wasm/py/windmill_parser_wasm.js"); + const { parse_python } = await import( + "../../wasm/py/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_python(content)); } else if (language === "nativets") { - const { parse_deno } = await import("../../wasm/ts/windmill_parser_wasm.js"); + const { parse_deno } = await import( + "../../wasm/ts/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_deno(content)); } else if (language === "bun") { - const { parse_deno } = await import("../../wasm/ts/windmill_parser_wasm.js"); + const { parse_deno } = await import( + "../../wasm/ts/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_deno(content)); } else if (language === "deno") { - const { parse_deno } = await import("../../wasm/ts/windmill_parser_wasm.js"); + const { parse_deno } = await import( + "../../wasm/ts/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_deno(content)); } else if (language === "go") { const { parse_go } = await import("../../wasm/go/windmill_parser_wasm.js"); @@ -599,7 +611,9 @@ export async function inferSchema( ...inferedSchema.args, ]; } else if (language === "postgresql") { - const { parse_sql } = await import("../../wasm/regex/windmill_parser_wasm.js"); + const { parse_sql } = await import( + "../../wasm/regex/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_sql(content)); inferedSchema.args = [ { name: "database", typ: { resource: "postgresql" } }, @@ -620,7 +634,9 @@ export async function inferSchema( ...inferedSchema.args, ]; } else if (language === "bash") { - const { parse_bash } = await import("../../wasm/regex/windmill_parser_wasm.js"); + const { parse_bash } = await import( + "../../wasm/regex/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_bash(content)); } else if (language === "powershell") { const { parse_powershell } = await import( @@ -628,10 +644,14 @@ export async function inferSchema( ); inferedSchema = JSON.parse(parse_powershell(content)); } else if (language === "php") { - const { parse_php } = await import("../../wasm/php/windmill_parser_wasm.js"); + const { parse_php } = await import( + "../../wasm/php/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_php(content)); } else if (language === "rust") { - const { parse_rust } = await import("../../wasm/rust/windmill_parser_wasm.js"); + const { parse_rust } = await import( + "../../wasm/rust/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_rust(content)); } else if (language === "csharp") { const { parse_csharp } = await import( @@ -647,12 +667,16 @@ export async function inferSchema( ); inferedSchema = JSON.parse(parse_ansible(content)); } else if (language === "java") { - const { parse_java } = await import("../../wasm/java/windmill_parser_wasm.js"); + const { parse_java } = await import( + "../../wasm/java/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_java(content)); } else if (language === "ruby") { - const { parse_ruby } = await import("../../wasm/ruby/windmill_parser_wasm.js"); + const { parse_ruby } = await import( + "../../wasm/ruby/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_ruby(content)); - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG } else { throw new Error("Invalid language: " + language); } @@ -743,10 +767,11 @@ export async function parseMetadataFile( path: string; workspaceRemote: Workspace; schemaOnly?: boolean; + globalDeps: GlobalDeps; + codebases: SyncCodebase[] }) | undefined, - globalDeps: GlobalDeps, - codebases: SyncCodebase[] + ): Promise<{ isJson: boolean; payload: any; path: string }> { let metadataFilePath = scriptPath + ".script.json"; try { @@ -777,13 +802,19 @@ export async function parseMetadataFile( ); metadataFilePath = scriptPath + ".script.yaml"; let scriptInitialMetadata = defaultScriptMetadata(); + const lockPath = scriptPath + ".script.lock"; + scriptInitialMetadata.lock = "!inline " + lockPath; const scriptInitialMetadataYaml = yamlStringify( scriptInitialMetadata as Record, yamlOptions ); + await Deno.writeTextFile(metadataFilePath, scriptInitialMetadataYaml, { createNew: true, }); + await Deno.writeTextFile(lockPath, "", { + createNew: true, + }); if (generateMetadataIfMissing) { log.info( @@ -798,8 +829,8 @@ export async function parseMetadataFile( generateMetadataIfMissing, false, false, - globalDeps, - codebases, + generateMetadataIfMissing.globalDeps, + generateMetadataIfMissing.codebases, false ); scriptInitialMetadata = (await yamlParseFile( @@ -889,6 +920,32 @@ export async function generateScriptHash( ); } +export async function clearGlobalLock(path: string): Promise { + const conf = await readLockfile(); + if (!conf?.locks) { + conf.locks = {}; + } + const isV2 = conf?.version == "v2"; + + if (isV2) { + // Remove the specific v2 lock entry + const key = v2LockPath(path); + if (conf.locks) { + Object.keys(conf.locks).forEach((k) => { + if (conf.locks) { + if (k.startsWith(key)) { + delete conf.locks[k]; + } + } + }); + } + await Deno.writeTextFile( + WMILL_LOCKFILE, + yamlStringify(conf as Record, yamlOptions) + ); + } +} + export async function updateMetadataGlobalLock( path: string, hash: string, @@ -901,7 +958,7 @@ export async function updateMetadataGlobalLock( const isV2 = conf?.version == "v2"; if (isV2) { - conf.locks[v2LockPath(path, hash)] = hash; + conf.locks[v2LockPath(path, subpath)] = hash; } else { if (subpath) { let prev: any = conf.locks[path]; diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index cb8b808725..d11b751a5b 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -208,3 +208,31 @@ export async function getIsWin(): Promise { } return isWin; } + +/** + * Writes content to a file only if it differs from existing content. + * Creates parent directories if they don't exist. + * + * @param path - The file path to write to + * @param content - The content to write + * @returns true if file was written, false if skipped (content unchanged) + */ +export function writeIfChanged(path: string, content: string): boolean { + try { + const existing = Deno.readTextFileSync(path); + if (existing === content) { + // console.log(`Content unchanged for ${path}`); + return false; // Content unchanged, skip write + } + } catch (error) { + // File doesn't exist or can't be read, proceed with write + if (!(error instanceof Deno.errors.NotFound)) { + // If it's not a "not found" error, we might want to know about it + // but still proceed with the write attempt + } + } + + // console.log(`Writing content to ${path}`); + Deno.writeTextFileSync(path, content); + return true; // File was written +} \ No newline at end of file diff --git a/cli/wasm/nu/windmill_parser_wasm.js b/cli/wasm/nu/windmill_parser_wasm.js index fc00a9ee3e..2f21f260da 100644 --- a/cli/wasm/nu/windmill_parser_wasm.js +++ b/cli/wasm/nu/windmill_parser_wasm.js @@ -56,13 +56,17 @@ function passStringToWasm0(arg, malloc, realloc) { return ptr; } -const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } ); +let cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } ); if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); }; +function decodeText(ptr, len) { + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} + function getStringFromWasm0(ptr, len) { ptr = ptr >>> 0; - return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); + return decodeText(ptr, len); } /** * @param {string} code diff --git a/cli/wasm/nu/windmill_parser_wasm_bg.js b/cli/wasm/nu/windmill_parser_wasm_bg.js new file mode 100644 index 0000000000..003c6ce549 --- /dev/null +++ b/cli/wasm/nu/windmill_parser_wasm_bg.js @@ -0,0 +1,125 @@ +let wasm; +export function __wbg_set_wasm(val) { + wasm = val; +} + + +let WASM_VECTOR_LEN = 0; + +let cachedUint8ArrayMemory0 = null; + +function getUint8ArrayMemory0() { + if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { + cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); + } + return cachedUint8ArrayMemory0; +} + +const lTextEncoder = typeof TextEncoder === 'undefined' ? (0, module.require)('util').TextEncoder : TextEncoder; + +const cachedTextEncoder = new lTextEncoder('utf-8'); + +const encodeString = (typeof cachedTextEncoder.encodeInto === 'function' + ? function (arg, view) { + return cachedTextEncoder.encodeInto(arg, view); +} + : function (arg, view) { + const buf = cachedTextEncoder.encode(arg); + view.set(buf); + return { + read: arg.length, + written: buf.length + }; +}); + +function passStringToWasm0(arg, malloc, realloc) { + + if (realloc === undefined) { + const buf = cachedTextEncoder.encode(arg); + const ptr = malloc(buf.length, 1) >>> 0; + getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf); + WASM_VECTOR_LEN = buf.length; + return ptr; + } + + let len = arg.length; + let ptr = malloc(len, 1) >>> 0; + + const mem = getUint8ArrayMemory0(); + + let offset = 0; + + for (; offset < len; offset++) { + const code = arg.charCodeAt(offset); + if (code > 0x7F) break; + mem[ptr + offset] = code; + } + + if (offset !== len) { + if (offset !== 0) { + arg = arg.slice(offset); + } + ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0; + const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len); + const ret = encodeString(arg, view); + + offset += ret.written; + ptr = realloc(ptr, len, offset, 1) >>> 0; + } + + WASM_VECTOR_LEN = offset; + return ptr; +} + +const lTextDecoder = typeof TextDecoder === 'undefined' ? (0, module.require)('util').TextDecoder : TextDecoder; + +let cachedTextDecoder = new lTextDecoder('utf-8', { ignoreBOM: true, fatal: true }); + +cachedTextDecoder.decode(); + +const MAX_SAFARI_DECODE_BYTES = 2146435072; +let numBytesDecoded = 0; +function decodeText(ptr, len) { + numBytesDecoded += len; + if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) { + cachedTextDecoder = new lTextDecoder('utf-8', { ignoreBOM: true, fatal: true }); + cachedTextDecoder.decode(); + numBytesDecoded = len; + } + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} + +function getStringFromWasm0(ptr, len) { + ptr = ptr >>> 0; + return decodeText(ptr, len); +} +/** + * @param {string} code + * @returns {string} + */ +export function parse_nu(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_nu(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +export function __wbindgen_init_externref_table() { + const table = wasm.__wbindgen_export_0; + const offset = table.grow(4); + table.set(0, undefined); + table.set(offset + 0, undefined); + table.set(offset + 1, null); + table.set(offset + 2, true); + table.set(offset + 3, false); + ; +}; + diff --git a/cli/wasm/nu/windmill_parser_wasm_bg.wasm b/cli/wasm/nu/windmill_parser_wasm_bg.wasm index 683fd3d380..638bb844f6 100644 Binary files a/cli/wasm/nu/windmill_parser_wasm_bg.wasm and b/cli/wasm/nu/windmill_parser_wasm_bg.wasm differ diff --git a/cli/wasm/regex/windmill_parser_wasm.d.ts b/cli/wasm/regex/windmill_parser_wasm.d.ts index 7ddef9498c..5ad7aef180 100644 --- a/cli/wasm/regex/windmill_parser_wasm.d.ts +++ b/cli/wasm/regex/windmill_parser_wasm.d.ts @@ -1,14 +1,14 @@ /* tslint:disable */ /* eslint-disable */ export function parse_bash(code: string): string; -export function parse_powershell(code: string): string; -export function parse_sql(code: string): string; -export function parse_mysql(code: string): string; -export function parse_oracledb(code: string): string; -export function parse_duckdb(code: string): string; -export function parse_bigquery(code: string): string; export function parse_snowflake(code: string): string; -export function parse_mssql(code: string): string; -export function parse_db_resource(code: string): string | undefined; -export function parse_graphql(code: string): string; export function parse_assets_sql(code: string): string; +export function parse_duckdb(code: string): string; +export function parse_graphql(code: string): string; +export function parse_db_resource(code: string): string | undefined; +export function parse_mysql(code: string): string; +export function parse_sql(code: string): string; +export function parse_oracledb(code: string): string; +export function parse_bigquery(code: string): string; +export function parse_mssql(code: string): string; +export function parse_powershell(code: string): string; diff --git a/cli/wasm/regex/windmill_parser_wasm.js b/cli/wasm/regex/windmill_parser_wasm.js index 0a47fb6ec8..ce0a45c4c4 100644 --- a/cli/wasm/regex/windmill_parser_wasm.js +++ b/cli/wasm/regex/windmill_parser_wasm.js @@ -56,13 +56,17 @@ function passStringToWasm0(arg, malloc, realloc) { return ptr; } -const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } ); +let cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } ); if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); }; +function decodeText(ptr, len) { + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} + function getStringFromWasm0(ptr, len) { ptr = ptr >>> 0; - return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); + return decodeText(ptr, len); } /** * @param {string} code @@ -83,120 +87,6 @@ export function parse_bash(code) { } } -/** - * @param {string} code - * @returns {string} - */ -export function parse_powershell(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_powershell(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_sql(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_sql(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_mysql(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_mysql(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_oracledb(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_oracledb(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_duckdb(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_duckdb(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_bigquery(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_bigquery(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - /** * @param {string} code * @returns {string} @@ -220,13 +110,51 @@ export function parse_snowflake(code) { * @param {string} code * @returns {string} */ -export function parse_mssql(code) { +export function parse_assets_sql(code) { let deferred2_0; let deferred2_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_mssql(ptr0, len0); + const ret = wasm.parse_assets_sql(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_duckdb(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_duckdb(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_graphql(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_graphql(ptr0, len0); deferred2_0 = ret[0]; deferred2_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); @@ -255,13 +183,13 @@ export function parse_db_resource(code) { * @param {string} code * @returns {string} */ -export function parse_graphql(code) { +export function parse_mysql(code) { let deferred2_0; let deferred2_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_graphql(ptr0, len0); + const ret = wasm.parse_mysql(ptr0, len0); deferred2_0 = ret[0]; deferred2_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); @@ -274,13 +202,89 @@ export function parse_graphql(code) { * @param {string} code * @returns {string} */ -export function parse_assets_sql(code) { +export function parse_sql(code) { let deferred2_0; let deferred2_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_assets_sql(ptr0, len0); + const ret = wasm.parse_sql(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_oracledb(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_oracledb(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_bigquery(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_bigquery(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_mssql(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_mssql(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_powershell(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_powershell(ptr0, len0); deferred2_0 = ret[0]; deferred2_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); diff --git a/cli/wasm/regex/windmill_parser_wasm_bg.wasm b/cli/wasm/regex/windmill_parser_wasm_bg.wasm index 67de5f24de..12a3a0f992 100644 Binary files a/cli/wasm/regex/windmill_parser_wasm_bg.wasm and b/cli/wasm/regex/windmill_parser_wasm_bg.wasm differ diff --git a/cli/wasm/regex/windmill_parser_wasm_bg.wasm.d.ts b/cli/wasm/regex/windmill_parser_wasm_bg.wasm.d.ts index 2219bb4518..f5d3f99bf1 100644 --- a/cli/wasm/regex/windmill_parser_wasm_bg.wasm.d.ts +++ b/cli/wasm/regex/windmill_parser_wasm_bg.wasm.d.ts @@ -1,18 +1,18 @@ /* tslint:disable */ /* eslint-disable */ export const memory: WebAssembly.Memory; +export const parse_assets_sql: (a: number, b: number) => [number, number]; export const parse_bash: (a: number, b: number) => [number, number]; -export const parse_powershell: (a: number, b: number) => [number, number]; -export const parse_sql: (a: number, b: number) => [number, number]; +export const parse_bigquery: (a: number, b: number) => [number, number]; +export const parse_db_resource: (a: number, b: number) => [number, number]; +export const parse_duckdb: (a: number, b: number) => [number, number]; +export const parse_graphql: (a: number, b: number) => [number, number]; +export const parse_mssql: (a: number, b: number) => [number, number]; export const parse_mysql: (a: number, b: number) => [number, number]; export const parse_oracledb: (a: number, b: number) => [number, number]; -export const parse_duckdb: (a: number, b: number) => [number, number]; -export const parse_bigquery: (a: number, b: number) => [number, number]; +export const parse_powershell: (a: number, b: number) => [number, number]; export const parse_snowflake: (a: number, b: number) => [number, number]; -export const parse_mssql: (a: number, b: number) => [number, number]; -export const parse_db_resource: (a: number, b: number) => [number, number]; -export const parse_graphql: (a: number, b: number) => [number, number]; -export const parse_assets_sql: (a: number, b: number) => [number, number]; +export const parse_sql: (a: number, b: number) => [number, number]; export const __wbindgen_export_0: WebAssembly.Table; export const __wbindgen_malloc: (a: number, b: number) => number; export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; diff --git a/cli/wasm/rust/windmill_parser_wasm.js b/cli/wasm/rust/windmill_parser_wasm.js index c0db59571d..6639224cf5 100644 --- a/cli/wasm/rust/windmill_parser_wasm.js +++ b/cli/wasm/rust/windmill_parser_wasm.js @@ -11,11 +11,7 @@ function getUint8ArrayMemory0() { return cachedUint8ArrayMemory0; } -const cachedTextEncoder = (typeof TextEncoder !== 'undefined' ? new TextEncoder('utf-8') : { encode: () => { throw Error('TextEncoder not available') } } ); - -const encodeString = function (arg, view) { - return cachedTextEncoder.encodeInto(arg, view); -}; +const cachedTextEncoder = new TextEncoder(); function passStringToWasm0(arg, malloc, realloc) { @@ -46,7 +42,7 @@ function passStringToWasm0(arg, malloc, realloc) { } ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0; const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len); - const ret = encodeString(arg, view); + const ret = cachedTextEncoder.encodeInto(arg, view); offset += ret.written; ptr = realloc(ptr, len, offset, 1) >>> 0; @@ -56,13 +52,17 @@ function passStringToWasm0(arg, malloc, realloc) { return ptr; } -const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } ); +let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); -if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); }; +cachedTextDecoder.decode(); + +function decodeText(ptr, len) { + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} function getStringFromWasm0(ptr, len) { ptr = ptr >>> 0; - return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); + return decodeText(ptr, len); } /** * @param {string} code @@ -99,23 +99,9 @@ const imports = { }; -const wasm_url = new URL('windmill_parser_wasm_bg.wasm', import.meta.url); -let wasmCode = ''; -switch (wasm_url.protocol) { - case 'file:': - wasmCode = await Deno.readFile(wasm_url); - break - case 'https:': - case 'http:': - wasmCode = await (await fetch(wasm_url)).arrayBuffer(); - break - default: - throw new Error(`Unsupported protocol: ${wasm_url.protocol}`); -} - -const wasmInstance = (await WebAssembly.instantiate(wasmCode, imports)).instance; -const wasm = wasmInstance.exports; -export const __wasm = wasm; +const wasmUrl = new URL('windmill_parser_wasm_bg.wasm', import.meta.url); +const wasm = (await WebAssembly.instantiateStreaming(fetch(wasmUrl), imports)).instance.exports; +export { wasm as __wasm }; wasm.__wbindgen_start(); diff --git a/cli/wasm/rust/windmill_parser_wasm_bg.wasm b/cli/wasm/rust/windmill_parser_wasm_bg.wasm index 1ad42fe519..bef68f9cc2 100644 Binary files a/cli/wasm/rust/windmill_parser_wasm_bg.wasm and b/cli/wasm/rust/windmill_parser_wasm_bg.wasm differ diff --git a/cli/wasm/ts/windmill_parser_wasm.d.ts b/cli/wasm/ts/windmill_parser_wasm.d.ts index 07c9bc242a..e21c8ebabf 100644 --- a/cli/wasm/ts/windmill_parser_wasm.d.ts +++ b/cli/wasm/ts/windmill_parser_wasm.d.ts @@ -2,5 +2,5 @@ /* eslint-disable */ export function parse_deno(code: string, main_override?: string | null): string; export function parse_outputs(code: string): string; -export function parse_ts_imports(code: string): string; export function parse_assets_ts(code: string): string; +export function parse_ts_imports(code: string): string; diff --git a/cli/wasm/ts/windmill_parser_wasm.js b/cli/wasm/ts/windmill_parser_wasm.js index d5fb34d262..20b7073aac 100644 --- a/cli/wasm/ts/windmill_parser_wasm.js +++ b/cli/wasm/ts/windmill_parser_wasm.js @@ -1,5 +1,27 @@ +let cachedUint8ArrayMemory0 = null; + +function getUint8ArrayMemory0() { + if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { + cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); + } + return cachedUint8ArrayMemory0; +} + +let cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } ); + +if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); }; + +function decodeText(ptr, len) { + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} + +function getStringFromWasm0(ptr, len) { + ptr = ptr >>> 0; + return decodeText(ptr, len); +} + function addToExternrefTable0(obj) { const idx = wasm.__externref_table_alloc(); wasm.__wbindgen_export_2.set(idx, obj); @@ -15,22 +37,9 @@ function handleError(f, args) { } } -const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } ); - -if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); }; - -let cachedUint8ArrayMemory0 = null; - -function getUint8ArrayMemory0() { - if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { - cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); - } - return cachedUint8ArrayMemory0; -} - -function getStringFromWasm0(ptr, len) { +function getArrayU8FromWasm0(ptr, len) { ptr = ptr >>> 0; - return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); + return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len); } function isLikeNone(x) { @@ -198,25 +207,6 @@ export function parse_outputs(code) { } } -/** - * @param {string} code - * @returns {string} - */ -export function parse_ts_imports(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_ts_imports(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - /** * @param {string} code * @returns {string} @@ -236,37 +226,56 @@ export function parse_assets_ts(code) { } } +/** + * @param {string} code + * @returns {string} + */ +export function parse_ts_imports(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_ts_imports(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + const imports = { __wbindgen_placeholder__: { - __wbg_buffer_609cc3eee51ed158: function(arg0) { - const ret = arg0.buffer; + __wbg_Error_1f3748b298f99708: function(arg0, arg1) { + const ret = Error(getStringFromWasm0(arg0, arg1)); return ret; }, - __wbg_call_672a4d21634d4a24: function() { return handleError(function (arg0, arg1) { + __wbg_call_2f8d426a20a307fe: function() { return handleError(function (arg0, arg1) { const ret = arg0.call(arg1); return ret; }, arguments) }, - __wbg_done_769e5ede4b31c67b: function(arg0) { + __wbg_done_4a7743b6f942c9f3: function(arg0) { const ret = arg0.done; return ret; }, - __wbg_entries_3265d4158b33e5dc: function(arg0) { + __wbg_entries_17f7acbc2d691c0d: function(arg0) { const ret = Object.entries(arg0); return ret; }, - __wbg_eval_d0dfcbbfaeff4b3c: function(arg0, arg1) { + __wbg_eval_12755dabbdfa08b1: function(arg0, arg1) { const ret = eval(getStringFromWasm0(arg0, arg1)); return ret; }, - __wbg_get_67b2ba62fc30de12: function() { return handleError(function (arg0, arg1) { + __wbg_get_27b4bcbec57323ca: function() { return handleError(function (arg0, arg1) { const ret = Reflect.get(arg0, arg1); return ret; }, arguments) }, - __wbg_get_b9b93047fe3cf45b: function(arg0, arg1) { + __wbg_get_59c6316d15f9f1d0: function(arg0, arg1) { const ret = arg0[arg1 >>> 0]; return ret; }, - __wbg_instanceof_ArrayBuffer_e14585432e3737fc: function(arg0) { + __wbg_instanceof_ArrayBuffer_59339a3a6f0c10ea: function(arg0) { let result; try { result = arg0 instanceof ArrayBuffer; @@ -276,7 +285,7 @@ const imports = { const ret = result; return ret; }, - __wbg_instanceof_Map_f3469ce2244d2430: function(arg0) { + __wbg_instanceof_Map_dd89a82d76d1b25f: function(arg0) { let result; try { result = arg0 instanceof Map; @@ -286,7 +295,7 @@ const imports = { const ret = result; return ret; }, - __wbg_instanceof_Uint8Array_17156bcf118086a9: function(arg0) { + __wbg_instanceof_Uint8Array_91f3c5adee7e6672: function(arg0) { let result; try { result = arg0 instanceof Uint8Array; @@ -296,77 +305,113 @@ const imports = { const ret = result; return ret; }, - __wbg_isArray_a1eab7e0d067391b: function(arg0) { + __wbg_isArray_bc2498eba6fcb71f: function(arg0) { const ret = Array.isArray(arg0); return ret; }, - __wbg_isSafeInteger_343e2beeeece1bb0: function(arg0) { + __wbg_isSafeInteger_6091d6e3ee1b65fd: function(arg0) { const ret = Number.isSafeInteger(arg0); return ret; }, - __wbg_iterator_9a24c88df860dc65: function() { + __wbg_iterator_96378c3c9a17347c: function() { const ret = Symbol.iterator; return ret; }, - __wbg_length_a446193dc22c12f8: function(arg0) { + __wbg_length_246fa1f85a0dea5b: function(arg0) { const ret = arg0.length; return ret; }, - __wbg_length_e2d2a49132c1b256: function(arg0) { + __wbg_length_904c0910ed998bf3: function(arg0) { const ret = arg0.length; return ret; }, - __wbg_new_a12002a7f91c75be: function(arg0) { + __wbg_new_9190433fb67ed635: function(arg0) { const ret = new Uint8Array(arg0); return ret; }, - __wbg_next_25feadfc0913fea9: function(arg0) { - const ret = arg0.next; - return ret; - }, - __wbg_next_6574e1a8a62d1055: function() { return handleError(function (arg0) { + __wbg_next_2e6b37020ac5fe58: function() { return handleError(function (arg0) { const ret = arg0.next(); return ret; }, arguments) }, - __wbg_set_65595bdd868b3009: function(arg0, arg1, arg2) { - arg0.set(arg1, arg2 >>> 0); + __wbg_next_3de8f2669431a3ff: function(arg0) { + const ret = arg0.next; + return ret; }, - __wbg_value_cd1ffa7b1ab794f1: function(arg0) { + __wbg_prototypesetcall_c5f74efd31aea86b: function(arg0, arg1, arg2) { + Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), arg2); + }, + __wbg_value_09d0b4eaab48b91d: function(arg0) { const ret = arg0.value; return ret; }, - __wbindgen_bigint_from_i64: function(arg0) { - const ret = arg0; - return ret; - }, - __wbindgen_bigint_from_u64: function(arg0) { - const ret = BigInt.asUintN(64, arg0); - return ret; - }, - __wbindgen_bigint_get_as_i64: function(arg0, arg1) { + __wbg_wbindgenbigintgetasi64_7637cb1a7fb9a81e: function(arg0, arg1) { const v = arg1; const ret = typeof(v) === 'bigint' ? v : undefined; getDataViewMemory0().setBigInt64(arg0 + 8 * 1, isLikeNone(ret) ? BigInt(0) : ret, true); getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true); }, - __wbindgen_boolean_get: function(arg0) { + __wbg_wbindgenbooleanget_59f830b1a70d2530: function(arg0) { const v = arg0; - const ret = typeof(v) === 'boolean' ? (v ? 1 : 0) : 2; - return ret; + const ret = typeof(v) === 'boolean' ? v : undefined; + return isLikeNone(ret) ? 0xFFFFFF : ret ? 1 : 0; }, - __wbindgen_debug_string: function(arg0, arg1) { + __wbg_wbindgendebugstring_bb652b1bc2061b6d: function(arg0, arg1) { const ret = debugString(arg1); const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len1 = WASM_VECTOR_LEN; getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); }, - __wbindgen_error_new: function(arg0, arg1) { - const ret = new Error(getStringFromWasm0(arg0, arg1)); + __wbg_wbindgenin_192b210aa1c401e9: function(arg0, arg1) { + const ret = arg0 in arg1; return ret; }, - __wbindgen_in: function(arg0, arg1) { - const ret = arg0 in arg1; + __wbg_wbindgenisbigint_7d76a1ca6454e439: function(arg0) { + const ret = typeof(arg0) === 'bigint'; + return ret; + }, + __wbg_wbindgenisfunction_ea72b9d66a0e1705: function(arg0) { + const ret = typeof(arg0) === 'function'; + return ret; + }, + __wbg_wbindgenisobject_dfe064a121d87553: function(arg0) { + const val = arg0; + const ret = typeof(val) === 'object' && val !== null; + return ret; + }, + __wbg_wbindgenjsvaleq_f27272c0a890df7f: function(arg0, arg1) { + const ret = arg0 === arg1; + return ret; + }, + __wbg_wbindgenjsvallooseeq_9dd7bb4b95ac195c: function(arg0, arg1) { + const ret = arg0 == arg1; + return ret; + }, + __wbg_wbindgennumberget_d855f947247a3fbc: function(arg0, arg1) { + const obj = arg1; + const ret = typeof(obj) === 'number' ? obj : undefined; + getDataViewMemory0().setFloat64(arg0 + 8 * 1, isLikeNone(ret) ? 0 : ret, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true); + }, + __wbg_wbindgenstringget_43fe05afe34b0cb1: function(arg0, arg1) { + const obj = arg1; + const ret = typeof(obj) === 'string' ? obj : undefined; + var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + var len1 = WASM_VECTOR_LEN; + getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); + }, + __wbg_wbindgenthrow_4c11a24fca429ccf: function(arg0, arg1) { + throw new Error(getStringFromWasm0(arg0, arg1)); + }, + __wbindgen_cast_4625c577ab2ec9ee: function(arg0) { + // Cast intrinsic for `U64 -> Externref`. + const ret = BigInt.asUintN(64, arg0); + return ret; + }, + __wbindgen_cast_9ae0607507abb057: function(arg0) { + // Cast intrinsic for `I64 -> Externref`. + const ret = arg0; return ret; }, __wbindgen_init_externref_table: function() { @@ -379,48 +424,6 @@ const imports = { table.set(offset + 3, false); ; }, - __wbindgen_is_bigint: function(arg0) { - const ret = typeof(arg0) === 'bigint'; - return ret; - }, - __wbindgen_is_function: function(arg0) { - const ret = typeof(arg0) === 'function'; - return ret; - }, - __wbindgen_is_object: function(arg0) { - const val = arg0; - const ret = typeof(val) === 'object' && val !== null; - return ret; - }, - __wbindgen_jsval_eq: function(arg0, arg1) { - const ret = arg0 === arg1; - return ret; - }, - __wbindgen_jsval_loose_eq: function(arg0, arg1) { - const ret = arg0 == arg1; - return ret; - }, - __wbindgen_memory: function() { - const ret = wasm.memory; - return ret; - }, - __wbindgen_number_get: function(arg0, arg1) { - const obj = arg1; - const ret = typeof(obj) === 'number' ? obj : undefined; - getDataViewMemory0().setFloat64(arg0 + 8 * 1, isLikeNone(ret) ? 0 : ret, true); - getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true); - }, - __wbindgen_string_get: function(arg0, arg1) { - const obj = arg1; - const ret = typeof(obj) === 'string' ? obj : undefined; - var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - var len1 = WASM_VECTOR_LEN; - getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); - getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); - }, - __wbindgen_throw: function(arg0, arg1) { - throw new Error(getStringFromWasm0(arg0, arg1)); - }, }, }; diff --git a/cli/wasm/ts/windmill_parser_wasm_bg.wasm b/cli/wasm/ts/windmill_parser_wasm_bg.wasm index 3e92fd399c..e4b9b2e87d 100644 Binary files a/cli/wasm/ts/windmill_parser_wasm_bg.wasm and b/cli/wasm/ts/windmill_parser_wasm_bg.wasm differ diff --git a/cli/wasm/ts/windmill_parser_wasm_bg.wasm.d.ts b/cli/wasm/ts/windmill_parser_wasm_bg.wasm.d.ts index 893670467d..9ffd962d51 100644 --- a/cli/wasm/ts/windmill_parser_wasm_bg.wasm.d.ts +++ b/cli/wasm/ts/windmill_parser_wasm_bg.wasm.d.ts @@ -1,10 +1,10 @@ /* tslint:disable */ /* eslint-disable */ export const memory: WebAssembly.Memory; +export const parse_assets_ts: (a: number, b: number) => [number, number]; export const parse_deno: (a: number, b: number, c: number, d: number) => [number, number]; export const parse_outputs: (a: number, b: number) => [number, number]; export const parse_ts_imports: (a: number, b: number) => [number, number]; -export const parse_assets_ts: (a: number, b: number) => [number, number]; export const __wbindgen_exn_store: (a: number) => void; export const __externref_table_alloc: () => number; export const __wbindgen_export_2: WebAssembly.Table; diff --git a/cli/windmill-utils-internal/package.json b/cli/windmill-utils-internal/package.json index 72ec2220e8..ff4418f79b 100644 --- a/cli/windmill-utils-internal/package.json +++ b/cli/windmill-utils-internal/package.json @@ -1,6 +1,6 @@ { "name": "windmill-utils-internal", - "version": "1.3.0", + "version": "1.3.1", "description": "Internal utility functions for Windmill", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -21,4 +21,4 @@ "files": [ "dist/**/*" ] -} +} \ No newline at end of file diff --git a/cli/windmill-utils-internal/src/inline-scripts/extractor.ts b/cli/windmill-utils-internal/src/inline-scripts/extractor.ts index be2d33c0de..0239301ccb 100644 --- a/cli/windmill-utils-internal/src/inline-scripts/extractor.ts +++ b/cli/windmill-utils-internal/src/inline-scripts/extractor.ts @@ -1,4 +1,4 @@ -import { newPathAssigner } from "../path-utils/path-assigner.ts"; +import { newPathAssigner, PathAssigner } from "../path-utils/path-assigner.ts"; import { FlowModule } from "../gen/types.gen.ts"; /** @@ -17,20 +17,24 @@ interface InlineScript { * * @param modules - Array of flow modules to process * @param mapping - Optional mapping of module IDs to custom file paths + * @param separator - Path separator to use * @param defaultTs - Default TypeScript runtime to use ("bun" or "deno") + * @param pathAssigner - Optional path assigner to reuse (for nested calls) * @returns Array of inline scripts with their paths and content */ export function extractInlineScripts( modules: FlowModule[], mapping: Record = {}, separator: string = "/", - defaultTs?: "bun" | "deno" + defaultTs?: "bun" | "deno", + pathAssigner?: PathAssigner ): InlineScript[] { - const pathAssigner = newPathAssigner(defaultTs ?? "bun"); + // Create pathAssigner only if not provided (top-level call), but reuse it for nested calls + const assigner = pathAssigner ?? newPathAssigner(defaultTs ?? "bun"); + return modules.flatMap((m) => { if (m.value.type == "rawscript") { - let basePath, ext; - [basePath, ext] = pathAssigner.assignPath(m.summary, m.value.language); + const [basePath, ext] = assigner.assignPath(m.summary, m.value.language); const path = mapping[m.id] ?? basePath + ext; const content = m.value.content; const r = [{ path: path, content: content }]; @@ -47,25 +51,39 @@ export function extractInlineScripts( m.value.modules, mapping, separator, - defaultTs + defaultTs, + assigner ); } else if (m.value.type == "branchall") { return m.value.branches.flatMap((b) => - extractInlineScripts(b.modules, mapping, separator, defaultTs) + extractInlineScripts(b.modules, mapping, separator, defaultTs, assigner) ); } else if (m.value.type == "whileloopflow") { return extractInlineScripts( m.value.modules, mapping, separator, - defaultTs + defaultTs, + assigner ); } else if (m.value.type == "branchone") { return [ ...m.value.branches.flatMap((b) => - extractInlineScripts(b.modules, mapping, separator, defaultTs) + extractInlineScripts( + b.modules, + mapping, + separator, + defaultTs, + assigner + ) + ), + ...extractInlineScripts( + m.value.default, + mapping, + separator, + defaultTs, + assigner ), - ...extractInlineScripts(m.value.default, mapping, separator, defaultTs), ]; } else { return []; diff --git a/cli/windmill-utils-internal/src/inline-scripts/replacer.ts b/cli/windmill-utils-internal/src/inline-scripts/replacer.ts index 9ec25e9f65..cad496a269 100644 --- a/cli/windmill-utils-internal/src/inline-scripts/replacer.ts +++ b/cli/windmill-utils-internal/src/inline-scripts/replacer.ts @@ -24,8 +24,8 @@ export async function replaceInlineScripts( localPath: string, separator: string = "/", removeLocks?: string[], - renamer?: (path: string, newPath: string) => void, - deleter?: (path: string) => void + // renamer?: (path: string, newPath: string) => void, + // deleter?: (path: string) => void ): Promise { await Promise.all(modules.map(async (module) => { if (!module.value) { @@ -34,7 +34,7 @@ export async function replaceInlineScripts( if (module.value.type === "rawscript" && module.value.content && module.value.content.startsWith("!inline")) { const path = module.value.content.split(" ")[1]; - const pathPrefix = path.split(".")[0]; + // const pathPrefix = path.split(".")[0]; const pathSuffix = path.split(".").slice(1).join("."); // new path is the module id with the same suffix const newPath = module.id + "." + pathSuffix; @@ -88,7 +88,8 @@ export async function replaceInlineScripts( try { module.value.lock = await fileReader(path.replaceAll("/", separator)); } catch { - logger.error(`Lock file ${path} not found`); + logger.error(`Lock file ${path} not found, treating as empty`); + module.value.lock = ""; } } } else if (module.value.type === "forloopflow" || module.value.type === "whileloopflow") { diff --git a/cli/windmill-utils-internal/src/parse/parse-schema.ts b/cli/windmill-utils-internal/src/parse/parse-schema.ts index a6417a1c01..9cd4a596b4 100644 --- a/cli/windmill-utils-internal/src/parse/parse-schema.ts +++ b/cli/windmill-utils-internal/src/parse/parse-schema.ts @@ -1,7 +1,7 @@ /** * Type alias for enum values - can be an array of strings or undefined */ -export type EnumType = string[] | undefined; +export type EnumType = string[] | { label: string; value: string }[] | undefined; /** * Represents a property in a JSON schema with various validation and display options @@ -17,7 +17,7 @@ export interface SchemaProperty { items?: { type?: "string" | "number" | "bytes" | "object" | "resource"; contentEncoding?: "base64"; - enum?: string[]; + enum?: EnumType; resourceType?: string; properties?: { [name: string]: SchemaProperty }; }; @@ -54,22 +54,22 @@ export function argSigToJsonSchemaType( | string | { resource: string | null } | { - list: - | (string | { name?: string; props?: { key: string; typ: any }[] }) - | { str: any } - | { object: { name?: string; props?: { key: string; typ: any }[] } } - | null; - } + list: + | (string | { name?: string; props?: { key: string; typ: any }[] }) + | { str: any } + | { object: { name?: string; props?: { key: string; typ: any }[] } } + | null; + } | { dynselect: string } | { dynmultiselect: string } | { str: string[] | null } | { object: { name?: string; props?: { key: string; typ: any }[] } } | { - oneof: { - label: string; - properties: { key: string; typ: any }[]; - }[]; - }, + oneof: { + label: string; + properties: { key: string; typ: any }[]; + }[]; + }, oldS: SchemaProperty ): void { const newS: SchemaProperty = { type: "" }; diff --git a/docker-compose.yml b/docker-compose.yml index f8831523ae..1770a1e4db 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,6 +46,7 @@ services: condition: service_healthy volumes: - worker_logs:/tmp/windmill/logs + logging: *default-logging windmill_worker: @@ -72,6 +73,9 @@ services: - /var/run/docker.sock:/var/run/docker.sock - worker_dependency_cache:/tmp/windmill/cache - worker_logs:/tmp/windmill/logs + # for AI agent memory + - worker_memory:/tmp/windmill/memory + logging: *default-logging ## This worker is specialized for "native" jobs. Native jobs run in-process and thus are much more lightweight than other jobs @@ -188,6 +192,7 @@ volumes: db_data: null worker_dependency_cache: null worker_logs: null + worker_memory: null windmill_index: null lsp_cache: null caddy_data: null diff --git a/docker/DockerfileFull b/docker/DockerfileFull index 1d5f639113..5ca64d5f51 100644 --- a/docker/DockerfileFull +++ b/docker/DockerfileFull @@ -1,18 +1,19 @@ FROM ghcr.io/windmill-labs/windmill:dev # Rust -COPY --from=rust:1.88.0 /usr/local/cargo /usr/local/cargo -COPY --from=rust:1.88.0 /usr/local/rustup /usr/local/rustup +COPY --from=rust:1.90.0 /usr/local/cargo /usr/local/cargo +COPY --from=rust:1.90.0 /usr/local/rustup /usr/local/rustup RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7 # Ansible RUN uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -s -t "$UV_TOOL_BIN_DIR/" || true # C# -COPY --from=bitnami/dotnet-sdk:9.0.101-debian-12-r0 /opt/bitnami/dotnet-sdk /opt/dotnet-sdk -RUN ln -s /opt/dotnet-sdk/bin/dotnet /usr/bin/dotnet -ENV DOTNET_ROOT="/opt/dotnet-sdk/bin" - +RUN wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh \ + && chmod +x dotnet-install.sh \ + && ./dotnet-install.sh --channel 9.0 --install-dir /usr/share/dotnet \ + && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet \ + && rm dotnet-install.sh # Nushell COPY --from=ghcr.io/nushell/nushell:0.101.0-bookworm /usr/bin/nu /usr/bin/nu diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index 376f994f89..cf420e1962 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -20,16 +20,18 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ FROM ghcr.io/windmill-labs/windmill-ee:dev # Rust -COPY --from=rust:1.88.0 /usr/local/cargo /usr/local/cargo -COPY --from=rust:1.88.0 /usr/local/rustup /usr/local/rustup +COPY --from=rust:1.90.0 /usr/local/cargo /usr/local/cargo +COPY --from=rust:1.90.0 /usr/local/rustup /usr/local/rustup RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7 # Ansible RUN uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -s -t "$UV_TOOL_BIN_DIR/" || true # dotnet SDK -COPY --from=bitnami/dotnet-sdk:9.0.101-debian-12-r0 /opt/bitnami/dotnet-sdk /opt/dotnet-sdk -RUN ln -s /opt/dotnet-sdk/bin/dotnet /usr/bin/dotnet -ENV DOTNET_ROOT="/opt/dotnet-sdk/bin" +RUN wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh \ + && chmod +x dotnet-install.sh \ + && ./dotnet-install.sh --channel 9.0 --install-dir /usr/share/dotnet \ + && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet \ + && rm dotnet-install.sh # Oracle DB Client COPY --from=oracledb-client /opt/oracle/23/lib /opt/oracle/23/lib diff --git a/docker/DockerfileNsjail b/docker/DockerfileNsjail index a995c76fac..26116499c9 100644 --- a/docker/DockerfileNsjail +++ b/docker/DockerfileNsjail @@ -51,9 +51,11 @@ RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7 RUN uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -s -t "$UV_TOOL_BIN_DIR/" || true # dotnet SDK -COPY --from=bitnami/dotnet-sdk:9.0.101-debian-12-r0 /opt/bitnami/dotnet-sdk /opt/dotnet-sdk -RUN ln -s /opt/dotnet-sdk/bin/dotnet /usr/bin/dotnet -ENV DOTNET_ROOT="/opt/dotnet-sdk/bin" +RUN wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh \ + && chmod +x dotnet-install.sh \ + && ./dotnet-install.sh --channel 9.0 --install-dir /usr/share/dotnet \ + && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet \ + && rm dotnet-install.sh # Oracle DB Client COPY --from=oracledb-client /opt/oracle/23/lib /opt/oracle/23/lib diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index ca749b02f0..2ee2d750d9 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -20,7 +20,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.15/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv -COPY --from=oven/bun:1.2.18 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.2.23 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 4b001f5d10..ba466307be 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -19,7 +19,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.15/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv -COPY --from=oven/bun:1.2.18 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.2.23 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/flake.nix b/flake.nix index 3bd150a614..102c79e8dd 100644 --- a/flake.nix +++ b/flake.nix @@ -50,11 +50,11 @@ xmlsec.dev libxslt.dev libclang.dev + libffi # For deno_ffi libtool nodejs postgresql pkg-config - glibc.dev clang cmake ]; @@ -103,6 +103,7 @@ wasm-pack deno emscripten + nushell # Needed for extra dependencies glibc_multi ]); @@ -231,17 +232,31 @@ set -e cd ./backend mkdir -p .minio-data/wmill - ${pkgs.minio}/bin/minio server ./.minio-data + ${pkgs.minio}/bin/minio server ./.minio-data --console-address ":9001" '') # Generate keys # TODO: Do not set new keys if ran multiple times (pkgs.writeScriptBin "wm-minio-keys" '' set -e cd ./backend + + # Set up MinIO alias ${pkgs.minio-client}/bin/mc alias set 'wmill-minio-dev' 'http://localhost:9000' 'minioadmin' 'minioadmin' - ${pkgs.minio-client}/bin/mc admin accesskey create 'wmill-minio-dev' | tee .minio-data/secrets.txt - echo "" - echo 'Saving to: ./backend/.minio-data/secrets.txt' + + # Check if secrets file exists and contains valid keys + if [[ -f .minio-data/secrets.txt ]] && [[ -s .minio-data/secrets.txt ]]; then + echo "Access keys already exist:" + cat .minio-data/secrets.txt + echo "" + echo "Keys loaded from: ./backend/.minio-data/secrets.txt" + else + echo "Creating new access keys..." + mkdir -p .minio-data + ${pkgs.minio-client}/bin/mc admin accesskey create 'wmill-minio-dev' | tee .minio-data/secrets.txt + echo "" + echo 'New keys saved to: ./backend/.minio-data/secrets.txt' + fi + echo "bucket: wmill" echo "endpoint: http://localhost:9000" '') @@ -298,9 +313,10 @@ # included we need to look in a few places. # See https://web.archive.org/web/20220523141208/https://hoverbear.org/blog/rust-bindgen-in-nix/ BINDGEN_EXTRA_CLANG_ARGS = - "${builtins.readFile "${stdenv.cc}/nix-support/libc-crt1-cflags"} ${ + # Prevent clang from using system headers - only use Nix headers + "-nostdinc ${builtins.readFile "${stdenv.cc}/nix-support/libc-crt1-cflags"} ${ builtins.readFile "${stdenv.cc}/nix-support/libc-cflags" - }${builtins.readFile "${stdenv.cc}/nix-support/cc-cflags"}${ + } ${builtins.readFile "${stdenv.cc}/nix-support/cc-cflags"} ${ builtins.readFile "${stdenv.cc}/nix-support/libcxx-cxxflags" } -idirafter ${pkgs.libiconv}/include ${ lib.optionalString stdenv.cc.isClang @@ -313,9 +329,10 @@ lib.getVersion stdenv.cc.cc } -isystem ${stdenv.cc.cc}/include/c++/${ lib.getVersion stdenv.cc.cc - }/${stdenv.hostPlatform.config} -idirafter ${stdenv.cc.cc}/lib/gcc/${stdenv.hostPlatform.config}/14.2.1/include" - }"; # NOTE: It is hardcoded to 14.2.1 -------------------------------------------------------------^^^^^^ - # Please update the version here as well if you want to update flake. + }/${stdenv.hostPlatform.config} -idirafter ${stdenv.cc.cc}/lib/gcc/${stdenv.hostPlatform.config}/${ + lib.getVersion stdenv.cc.cc + }/include" + }"; }; packages.default = self.packages.${system}.windmill; packages.windmill-client = pkgs.buildNpmPackage { diff --git a/frontend/CLAUDE.md b/frontend/CLAUDE.md index 6aa8ef3c48..0596b62443 100644 --- a/frontend/CLAUDE.md +++ b/frontend/CLAUDE.md @@ -15,42 +15,13 @@ - **Use Windmill's theming classes** for consistent colors and surfaces - **Avoid custom styles** - prefer Tailwind utility classes - **Follow existing patterns** - look at other components for reference +- **Respect design guidelines** - rules are defined in 'brand-guidelines.md' -### Windmill Theme Classes +### UI Components -Use these semantic color classes that automatically handle light/dark modes: - -#### Backgrounds -- `bg-surface` - Main surface background -- `bg-surface-secondary` - Secondary/elevated surfaces -- `bg-surface-hover` - Hover states for interactive elements - -#### Text Colors -- `text-primary` - Primary text color -- `text-secondary` - Secondary text (less prominent) -- `text-tertiary` - Tertiary text (subtle/muted) - -#### Borders -- `border-gray-200 dark:border-gray-700` - Standard borders that adapt to theme - -#### Status Colors -Use standard Tailwind color classes with dark mode variants: -- Success: `text-green-500`, `bg-green-100 dark:bg-green-900/30` -- Error: `text-red-500`, `bg-red-50 dark:bg-red-900/20` -- Warning: `text-yellow-500`, `bg-yellow-100 dark:bg-yellow-900/30` -- Info: `text-blue-500`, `bg-blue-100 dark:bg-blue-900/30` - -#### Typography -- `font-mono` - For code/technical content -- `text-xs`, `text-sm`, `text-2xs` - Standard text sizes -- Use `font-medium`, `font-semibold` for emphasis - -### Layout Guidelines - -- Use Tailwind spacing utilities (`p-3`, `m-2`, `gap-2`, etc.) -- Use flexbox/grid utilities for layouts -- Use `transition-colors` for smooth hover effects -- Use `overflow-hidden`, `rounded-md` for consistent card styles +- Use the component TextInput for all text inputs +- Form components (TextInputs, ToggleButtons, Select ...) should all use the same size when put together, using the unified size system. +- Read carefully components props JSDoc before using them ## Backend API @@ -115,3 +86,27 @@ AuditService.listAuditLogs({ operations?: string // from operations parameter }) ``` + +## Svelte 5 documentation + +You are able to use the Svelte MCP server, where you have access to comprehensive Svelte 5 and SvelteKit documentation. Here's how to use the available tools effectively: + +### 1. list-sections + +Use this FIRST to discover all available documentation sections. Returns a structured list with titles, use_cases, and paths. +When asked about Svelte or SvelteKit topics, ALWAYS use this tool at the start of the chat to find relevant sections. + +### 2. get-documentation + +Retrieves full documentation content for specific sections. Accepts single or multiple sections. +After calling the list-sections tool, you MUST analyze the returned documentation sections (especially the use_cases field) and then use the get-documentation tool to fetch ALL documentation sections that are relevant for the user's task. + +### 3. svelte-autofixer + +Analyzes Svelte code and returns issues and suggestions. +You MUST use this tool whenever writing Svelte code before sending it to the user. Keep calling it until no issues or suggestions are returned. + +### 4. playground-link + +Generates a Svelte Playground link with the provided code. +After completing the code, ask the user if they want a playground link. Only call this tool after user confirmation and NEVER if code was written to files in their project. diff --git a/frontend/README_DEV.md b/frontend/README_DEV.md index c17ac944c8..c5ba2a0c15 100644 --- a/frontend/README_DEV.md +++ b/frontend/README_DEV.md @@ -139,15 +139,6 @@ If you develop wasm parser for new language you can also pass `--wasm-pkg > ~/.zshrc - source ~/.zshrc - ``` - In the root folder: ```bash @@ -166,6 +157,20 @@ In the frontend folder: REMOTE=http://127.0.0.1:8000 REMOTE_LSP=http://127.0.0.1:3001 npm run dev ``` +**Known issue on M1 Mac while running `cargo run`** + +- You may encounter `linking with cc failed` build time error. +- To solve this run: + ```bash + echo 'export RUSTFLAGS="-L/opt/homebrew/opt/libomp/lib"' >> ~/.zshrc + source ~/.zshrc + ``` + +**Known issue on M1 Mac while running `cargo run` with the `deno_core` feature** + +- You may encounter ``failed to run custom build command for `libffi-sys v2.3.0` `` build time error. +- To solve this use the `deno_core_mac` feature flag _instead_ of `deno_core`. You might need to install `libffi` (e.g. `brew install libffi`). + ### Formatting This project uses [prettier](https://prettier.io/docs/en/install.html) and diff --git a/frontend/brand-guidelines.md b/frontend/brand-guidelines.md new file mode 100644 index 0000000000..38eaa588c5 --- /dev/null +++ b/frontend/brand-guidelines.md @@ -0,0 +1,815 @@ +# Windmill Brand Guidelines + +_This document contains the complete brand guidelines for Windmill, including visual identity, design system, and communication standards._ + +# Voice & Communication + +This section defines how your brand communicates across all channels and touchpoints. + +# Tone of Voice + +Your tone of voice should match the serious, professional, no-nonsense character of the product and brand. + +## Suggested Content Structure + +| **Approach** | **✅ Do This** | **❌ Not This** | +| ----------------------------------------------------------------------------- | ------------------------------------------------------- | ------------------------------------------------------- | +| **Direct & Honest** – Say exactly what the product does, no marketing fluff. | "Deploy complex workflows in minutes." | "Experience seamless automation magic." | +| **Technical & Precise** – Speak like your audience: developers and engineers. | "Scale scripts without worrying about dependencies." | "Your team will love our easy drag-and-drop interface." | +| **Confident & Assertive** – Show that the product can handle anything. | "No limitations. Every workflow is fully customizable." | "Works well for most use cases." | +| **Minimalist & Functional** – Avoid unnecessary adjectives or filler words. | | | + +# Visual Identity + +This section covers all visual elements that make up your brand identity. + +## Overview + +Visual Identity includes: + +- Logo: Usage guidelines, variations, and spacing requirements +- Color System: Primary, secondary, and semantic colors with specifications +- Typography: Font families, hierarchy, and implementation guidelines + Use the navigation on the left to explore each subsection and add your specific visual identity content. + +# Color system + +Our color system is designed for **reliability, clarity, and trust**. Windmill is destined to be used by big companies, so colors must convey a sense of professionalism and seriousness. We stay away from vibrant colors in the application interface and use monochromatic tones with purposeful accent colors. + +## Design Principles + +- **Meaningful, not decorative**: Every color serves a functional purpose +- **Reliability over vibrancy**: Muted tones convey trust and professionalism +- **Context-aware**: Different palettes for app vs marketing contexts +- **Accessible**: All colors meet WCAG AA contrast requirements +- **Consistent**: Systematic approach to color usage across all interfaces + +## Color Philosophy + +**Colors are meaningful and never decorative.** We distinguish between: + +- **App palette**: Less vibrant colors for the core application interface +- **Web palette**: More vibrant colors for marketing and brand recognition +- **Monochromatic scale**: Nord-based neutrals for surfaces and backgrounds + +## Color Palette Overview + +Our complete color system in action, showing how all color categories work together across light and dark themes: +![Complete Windmill color palette (Light mode)](./static/brand-guidelines-assets/example-color-palette-light.svg) +_Light mode example_ + +## Accent Colors + +The **luminance-blue** is our primary accent color, used throughout the app for interactive elements, active states, and user actions. +_[Color palette display - see original documentation for interactive colors]_ +**When to use:** + +- Call-to-action buttons +- Active navigation items +- Toggle switches (on state) +- Progress bars +- Interactive links +- Selected states + +## Surface Colors + +Surface colors create depth and layout structure with clear hierarchy between different interface levels. +_[Color palette display - see original documentation for interactive colors]_ + +## Text Colors + +Text colors provide clear hierarchy and readability across light and dark themes. +_[Color palette display - see original documentation for interactive colors]_ + +## Border Colors + +Border colors define element separation and structure. +_[Color palette display - see original documentation for interactive colors]_ + +## Feedback Colors + +Standard semantic colors for system feedback and status communication. +_[Color palette display - see original documentation for interactive colors]_ + +## Reserved Colors + +These colors are exclusively reserved for specific features and should not be used elsewhere. +_[Color palette display - see original documentation for interactive colors]_ +**Usage:** + +- AI-powered script generation +- Magic wand icons +- AI assistance features +- Smart suggestions + +## Web/Marketing Colors + +More vibrant colors used exclusively for marketing materials and the website (not in the core application). +_[Color palette display - see original documentation for interactive colors]_ +**Important:** These colors should **never** be used in the core application interface. They are reserved for: + +- Marketing website +- Landing pages +- Documentation site headers +- Brand materials +- Social media assets + +## Color Reference + +Complete color system with usage guidelines, hex values, and Tailwind classes: +**Implementation Rule:** Always use the provided Tailwind classes in your components. Never use hex values directly in styles - this ensures consistency and theme switching compatibility. + +```jsx +// ✅ Correct - use Tailwind classes +Save +Content +// ❌ Wrong - don't use hex values +Save +``` + +## Do's and Don'ts + +### ✅ Do + +- Use provided Tailwind classes for all color implementations +- Use `accent-primary` sparingly for important actions +- Rely on surface colors for most interface backgrounds +- Apply text colors according to content hierarchy +- Use proper border colors for element separation +- Apply feedback colors consistently for their semantic meaning +- Test color combinations for accessibility compliance +- Reserve AI purple exclusively for AI features +- Follow the defined color token structure + +### ❌ Don't + +- Use hex values directly in component styles or CSS +- Mix web/marketing colors with app interface colors +- Use AI purple for non-AI features +- Create custom color variations between defined tokens +- Use color alone to convey meaning (pair with icons/text) +- Apply accent colors to large surface areas +- Use marketing blue (`#3B82F6`) in the app interface +- Use `accent-primary` for large backgrounds +- Mix different color token categories inappropriately + +### Color-Blind Considerations + +- Never use color alone to convey information +- Pair color with icons, text, or patterns +- Test designs with color-blind simulation tools +- Ensure sufficient contrast in monochrome + +## Quick Reference + +| Token | Light Mode | Dark Mode | Tailwind Class | Usage | +| ------------------------ | ---------- | --------- | -------------------------- | ------------------------------------------------------ | +| **Accent Colors** | | | | | +| accent-primary | #758ff8 | #7085db | `accent-primary` | Primary accent color for buttons, links, active states | +| accent-hover | #5074f6 | #5670d5 | `accent-hover` | Hover state for interactive accent elements | +| accent-clicked | #2c5beb | #425bbd | `accent-clicked` | Active/pressed state for accent elements | +| accent-secondary | #293676 | #e8ebfb | `accent-secondary` | Secondary accent for strong emphasis | +| accent-secondary-hover | #1e255f | #c3c9df | `accent-secondary-hover` | Hover state for accent secondary elements | +| accent-secondary-clicked | #303f82 | #9da6ca | `accent-secondary-clicked` | Active/pressed state for accent secondary elements | +| accent-selected | #bfdbfe4c | #6790c44c | `accent-selected` | Selected state background | +| | | | | | +| **Surface Colors** | | | | | +| surface-primary | #fbfbfd | #2e3441 | `surface-primary` | Main application background | +| surface-secondary | #efeff4 | #272c35 | `surface-secondary` | Secondary backgrounds, sections | +| surface-tertiary | #ffffff | #353c4a | `surface-tertiary` | Cards, modals, elevated surfaces | +| surface-hover | #cfcfe233 | #7784a119 | `surface-hover` | Hover states for neutral elements | +| surface-selected | #ffffff | #434c5e | `surface-selected` | Selected neutral elements | +| surface-disabled | #d8d8e433 | #212732 | `surface-disabled` | Disabled elements, inactive states | +| surface-sunken | #e8e8ef | #242832 | `surface-sunken` | Sunken or inset surfaces | +| surface-input | #ffffff | #292e38 | `surface-input` | Input field backgrounds | +| | | | | | +| **Text Colors** | | | | | +| text-primary | #3d4758 | #d4d7dd | `text-primary` | Default text, body content | +| text-emphasis | #1d2430 | #eeeff2 | `text-emphasis` | Headers, labels, emphasized content | +| text-secondary | #718096 | #a9b0ba | `text-secondary` | Supporting information, metadata | +| text-tertiary | #505c70 | #a8aeb7 | `text-tertiary` | Subtle text, captions | +| text-hint | #8d93a1 | #8d93a1 | `text-hint` | Placeholders, tooltips, hints | +| text-disabled | #a0aec0 | #9098a2 | `text-disabled` | Disabled states, unavailable options | +| text-accent | #5074f6 | #c7cefc | `text-accent` | Accent colored text, links | +| | | | | | +| **Border Colors** | | | | | +| border-light | #e5e7eb | #374457 | `border-light` | Subtle borders, dividers | +| border-normal | #9ca3af | #a9b0ba | `border-normal` | Standard borders, form inputs | +| border-accent | #2c5beb | #a0affa | `border-accent` | Accent borders, focus states | +| border-selected | #a0affa | #6475b7 | `border-selected` | Selected element borders | +| | | | | | +| **Reserved Colors** | | | | | +| ai-primary | #a02cde | #f0c6fb | `ai-primary` | AI features, magic wand icon, AI-powered functionality | +| | | | | | +| **Feedback Colors** | | | | | +| success | #22c55e | #22c55e | `green-500` | Success states, positive feedback, completed actions | +| warning | #eab308 | #eab308 | `yellow-500` | Warning states, caution messages, pending actions | +| error | #ef4444 | #ef4444 | `red-500` | Error states, failed actions, destructive operations | +| info | #3b82f6 | #3b82f6 | `blue-500` | Information states, neutral notifications | +| | | | | | + +Remember: **Colors are meaningful, not decorative.** Every color choice should serve a clear functional purpose in the user interface. + +# Elevation + +Windmill uses a **minimal elevation system** based on surface colors and strategic shadows. We prioritize clarity and simplicity over complex layering effects. + +## Elevation Principles + +- **Surface colors create depth**: Darker surfaces appear deeper than lighter ones +- **Shadows only for overlays and movement**: Not for making elements stand out +- **Light borders for grouping**: Preferred over shadows for content separation +- **Use sparingly**: Limit elevation to avoid visual noise + +## Surface Depth System + +We use surface colors from our color system to create depth hierarchy: + +- **`surface-primary`** (#FBFBFD): Default elevation, main backgrounds +- **`surface-secondary`** (#EFEFF4): Sunken surfaces, recessed areas +- **`surface-tertiary`** (#FFFFFF): Elevated cards and content areas + +## Shadow Usage + +### When to Use Shadows + +**✅ Use shadows for:** + +- **Overlays**: Modals, dropdowns, tooltips (`shadow-lg`) +- **Moving elements**: Drag and drop, active states (`shadow-md`) + **❌ Don't use shadows for:** +- Making buttons or elements stand out +- Content grouping or separation +- Decorative purposes +- Permanent interface elements + +### Shadow Specifications + +- **`shadow-md`**: For moving elements and temporary elevation +- **`shadow-lg`**: For overlays and floating content +- **Light borders**: `border-light` (#E5E7EB) for grouping instead of shadows + +## Examples + +### Overlay Elevation + +Modals, dropdowns, and floating content use `shadow-lg` with light borders: +![Overlay elevation example showing modal with shadow](./static/brand-guidelines-assets/elevation-overlay.svg) + +### Sunken Surface + +Recessed areas use `surface-secondary` to appear deeper than the main background: +![Sunken surface example showing recessed area](./static/brand-guidelines-assets/elevation-sunken.svg) + +## Do's and Don'ts + +### ✅ Do + +- Use `surface-secondary` for sunken or recessed areas +- Apply `shadow-lg` only to overlays (modals, dropdowns) +- Use `shadow-md` for elements being moved or dragged +- Prefer light borders (`border-light`) for content grouping +- Keep elevation simple and purposeful +- Use `surface-tertiary` for elevated cards when needed + +### ❌ Don't + +- Use shadows to make buttons or static elements stand out +- Combine multiple elevation techniques unnecessarily +- Create custom shadow values outside the system +- Use elevation purely for decoration +- Apply heavy shadows that distract from content +- Overuse elevation effects throughout the interface + Remember: **Elevation should enhance usability, not create visual complexity.** When in doubt, use surface colors instead of shadows for depth. + +# Typography + +Our typography system is designed for **clarity, efficiency, and minimalism**. With limited screen space in a complex developer tool, we prioritize readability and information density over decorative hierarchy. + +## Design Principles + +- **Space-efficient**: Default to 12px for body text to maximize content visibility +- **Minimal hierarchy**: Only 3 header levels to avoid confusion +- **Weight over size**: Use font-weight to create hierarchy, not excessive size changes +- **Functional**: Every style has a clear, specific purpose + +## Font Family + +We use **Inter** for all UI text due to its excellent readability at small sizes and professional appearance. +For technical content (IDs, code snippets, JSON), use the system **monospace** font. + +## Text Colors + +_[Color palette display - see original documentation for interactive colors]_ + +### Color Philosophy + +We use a **lighter default** with a **darker emphasis option** to: + +- Reduce eye strain during extended use +- Create clear visual hierarchy without relying on size +- Make emphasized content truly stand out +- Align with modern developer tool aesthetics + **Key principle:** Use `text-primary` for most content, reserve `text-emphasis` for true importance. + +## Type Styles + +### Typography Scale Overview + +### App Page Title + +**When to use**: Main application page titles, primary navigation headings + +```tsx +class = 'font-semibold text-2xl text-emphasis'; +``` + +- **Size**: 24px (`text-2xl`) +- **Weight**: 600 (`font-semibold`) +- **Color**: `text-emphasis` +- **Example**: "Job Orchestrator", "Flow Builder", "Resource Manager" + +--- + +### Page Title + +**When to use**: Top-level page headings, modal titles, main view names + +```tsx +class = 'text-lg font-semibold text-emphasis'; +``` + +- **Size**: 18px (`text-lg`) +- **Weight**: 600 (`font-semibold`) +- **Color**: `text-emphasis` +- **Example**: "Job Orchestrator Dashboard", "Edit Flow Configuration" + +--- + +### Section Header + +**When to use**: Panel headers, card titles, collapsible section names, sidebar groups + +```tsx +class = 'text-sm font-semibold text-emphasis'; +``` + +- **Size**: 14px (`text-sm`) +- **Weight**: 600 (`font-semibold`) +- **Color**: `text-emphasis` +- **Example**: "Active Jobs", "Configuration", "Environment Variables" + +--- + +### Body + +**When to use**: Default text throughout the application - descriptions, content, list items, table cells + +```tsx +class = 'text-xs font-normal text-primary'; +``` + +- **Size**: 12px (`text-xs`) +- **Weight**: 400 (`font-normal`) +- **Color**: `text-primary` +- **Example**: Form descriptions, paragraph content, dialog text + **This is your default**. When in doubt, use this style. + +--- + +### Body Emphasized + +**When to use**: Important labels, form field labels, tab labels, emphasis within body text + +```tsx +class = 'text-xs font-semibold text-emphasis'; +``` + +- **Size**: 12px (`text-xs`) +- **Weight**: 600 (`font-semibold`) +- **Color**: `text-emphasis` +- **Example**: "Job Name:", "Status:", button labels + +--- + +### Secondary Text + +**When to use**: Supporting information, metadata, timestamps, status descriptions + +```tsx +class = 'text-xs font-normal text-secondary'; +``` + +- **Size**: 12px (`text-xs`) +- **Weight**: 400 (`font-normal`) +- **Color**: `text-secondary` +- **Example**: "Last run 2 hours ago", "Created by John Doe", file sizes + +--- + +### Caption + +**When to use**: Helper text below inputs, table column headers, inline annotations, badges + +```tsx +class = 'text-2xs font-normal text-secondary'; +``` + +- **Size**: 11px (`text-2xs`) +- **Weight**: 400 (`font-normal`) +- **Color**: `text-secondary` +- **Example**: "Optional field", "Max 100 characters", column headers + +--- + +### Hint + +**When to use**: Input placeholders, tooltip content, empty state messages, subtle guidance + +```tsx +class = 'text-2xs font-normal text-hint'; +``` + +- **Size**: 11px (`text-2xs`) +- **Weight**: 400 (`font-normal`) +- **Color**: `text-hint` +- **Example**: "Enter job name...", "Search flows", tooltip text + +--- + +### Code/Monospace + +**When to use**: Job IDs, code snippets, file paths, API endpoints, JSON keys, technical identifiers + +```tsx +class = 'text-2xs font-mono font-normal text-emphasis'; +``` + +- **Size**: 11px (`text-2xs`) +- **Weight**: 400 (`font-normal`) +- **Color**: `text-emphasis` +- **Font**: System monospace +- **Example**: `job_id_12345`, `/api/v1/jobs`, `ENV_VAR_NAME` + **Note**: Use `text-emphasis` for code to ensure technical values stand out and are easily scannable. + +--- + +## Usage Guidelines + +### Creating Hierarchy + +Use these methods in order of preference: + +1. **Font weight** - Semibold (600) for headers and emphasis, normal (400) for body +2. **Color** - Primary for main content, secondary for supporting info, hint for subtle guidance +3. **Size** - Only change size for true hierarchy levels (page title vs section header vs body) + **Don't** create hierarchy by: + +- Making text larger than 24px (except for App Page Titles) +- Using more than 3 header levels +- Adding excessive spacing or borders + +## Creating Visual Hierarchy - Priority Order + +1. **Font Weight** - Use semibold (600) for emphasis +2. **Color** - Use textEmphasis for important content +3. **Position & Spacing** - Group related content, add white space +4. **Size** - Only use defined type styles, never custom sizes + +### ❌ Don't + +- Increase font size to make something "stand out" +- Create one-off font sizes for special cases +- Use large headers in dense UI areas + +### ✅ Do + +- Use font-weight to emphasize within the same size +- Use textEmphasis color for important content +- Add spacing around important elements + +## Text Casing + +### Primary Rule: Use Sentence Case + +Use sentence case for all UI text—capitalize only the first word and proper nouns. This approach improves readability and is faster to implement consistently. +**Examples:** + +- ✅ "Create new flow" +- ✅ "Edit Windmill resource" +- ❌ "Create New Flow" +- ❌ "SAVE CHANGES" + +### Casing by Component Type + +**Page titles and headings** + +- Use sentence case: "Job orchestrator dashboard" + **Buttons and actions** +- Use sentence case: "Save changes", "Delete job" + **Form labels** +- Use sentence case: "Job name", "Resource type" + **Navigation items** +- Use sentence case: "User settings", "Resource manager" + **Error messages and notifications** +- Use sentence case: "Job completed successfully" + +### Always Capitalize + +- **Proper nouns**: Windmill, Docker, Python, GitHub +- **Acronyms**: API, HTTP, JSON, SQL +- **First word** of any sentence or UI element + +### Special Cases + +- **Technical identifiers**: Keep original casing (`job_id_123`, `ENV_VAR`) +- **Brand names**: Follow brand guidelines (iPhone, macOS) +- **Abbreviations**: Use standard forms (ID, URL, vs.) + +### Accessibility Note + +Avoid ALL CAPS text except for very short labels (2-3 characters max). All caps text is slower to read and can appear aggressive to users. + +## Decision Tree + +Not sure which style to use? Follow this: +Is it a main application page title? +→ **Yes**: App Page Title (24px, semibold) +→ **No**: Continue +Is it a page or modal title? +→ **Yes**: Page Title (18px, semibold) +→ **No**: Continue +Is it a section/panel header? +→ **Yes**: Section Header (14px, semibold) +→ **No**: Continue +Is it technical data (ID, code, path)? +→ **Yes**: Code/Monospace (11px, mono) +→ **No**: Continue +Is it a placeholder or tooltip? +→ **Yes**: Hint (11px, hint color) +→ **No**: Continue +Is it helper text or a table header? +→ **Yes**: Caption (11px, secondary color) +→ **No**: Continue +Is it metadata or supporting info? +→ **Yes**: Secondary Text (12px, secondary color) +→ **No**: Continue +Is it a label or needs emphasis? +→ **Yes**: Body Emphasized (12px, semibold weight, emphasis color) +→ **No**: Body (12px, normal, primary color) ← **DEFAULT** + +## Accessibility + +- All text colors meet contrast requirements on standard backgrounds +- Never use font size alone to convey meaning +- Ensure disabled text (`text-disabled`) is paired with visual disabled states + +## Common Mistakes + +❌ **Don't** create custom font sizes between defined styles +✅ **Do** use the defined type styles +❌ **Don't** use Section Header inside table cells +✅ **Do** use Caption for table headers +❌ **Don't** use Body Emphasized everywhere for "importance" +✅ **Do** reserve it for labels and truly emphasized content +❌ **Don't** make job IDs or code bold/colored +✅ **Do** use monospace font with text-emphasis for technical identifiers +❌ **Don't** use text-emphasis for body paragraphs +✅ **Do** use text-primary for most content, text-emphasis for headers/labels + +# Design System + +This section covers the complete design system including components, layouts, and interaction patterns. + +## Overview + +Design System includes: + +- Iconography: Icon library and usage guidelines +- Spacing & Grid: Layout fundamentals and spacing scales +- Components: Reusable UI components and specifications +- Layout: Page structure and content organization principles + Use the navigation on the left to explore each subsection and add your specific design system content. + +# Components + +## Core Rules + +### 1. Always Use the Component Library + +**Never create custom components.** Use only the provided components from Windmill's library. If you need functionality that doesn't exist, request it from the design system team. + +### 2. No Style Hacking + +**Do not override component styles.** Components provide props for all supported variants and configurations. If you need a different appearance, use the appropriate prop variant. + +```jsx +// ✅ Correct - use provided variants +// ❌ Wrong - don't add custom styles +``` + +### 3. Check Guidelines First + +**Always consult these component guidelines** before implementing. Each component section specifies: + +- When to use each variant +- Proper implementation patterns +- Accessibility requirements +- Common mistakes to avoid + +## Quick Reference + +### Before You Code + +1. Check if a component exists for your use case +2. Read the component's specific guidelines +3. Use only the documented props and variants +4. Test accessibility with keyboard navigation + +## Buttons + +Windmill uses **4 button types** with clear hierarchy. Each button type comes in 3 variants (text, icon+text, icon-only) and supports multiple states including hover, active, disabled, and selected (default and subtle only). +**Button Hierarchy:** + +- **Accent Secondary** (Highest Priority): Main conversion CTAs on landing pages - "Sign up", "Get started", "Download" +- **Accent** (High Priority): Most important action per view - "Save", "Submit", "Create" (only one per screen) +- **Default** (Standard Priority): Secondary actions and most UI interactions - "Cancel", "Edit", "Delete" +- **Subtle** (Low Priority): Tertiary actions in dense interfaces - toolbars, button groups + **Button States:** +- **Default**: Standard button appearance +- **Hover**: Interactive feedback when cursor hovers over button +- **Active/Pressed**: Visual feedback when button is clicked or pressed +- **Disabled**: Non-interactive state for unavailable actions +- **Selected**: Active selection state (available only for Default and Subtle variants) + **Usage Rules:** +- ✅ Use appropriate hierarchy, provide tooltips for icon-only buttons, test all states including disabled and selected +- ❌ Don't use multiple Accent buttons per view, use Accent Secondary outside marketing, create custom styles, or use selected state on Accent variants + +## Implementation Notes + +When implementing any component: + +1. **Follow the design system**: Use only the components and variants documented here +2. **Accessibility first**: All components include built-in accessibility features +3. **Test thoroughly**: Verify functionality across different states and themes +4. **Ask questions**: When in doubt, consult the design system team before creating custom solutions + +# Iconography + +We use the **[Lucide icon library](https://lucide.dev/)** to ensure a consistent, modern, and lightweight visual language. Icons are line-only, aligning with our clean and technical aesthetic. + +## Do's and Don'ts + +### ✅ Do + +- Use Lucide icons consistently throughout the interface +- Maintain the original 2px stroke width and style +- Use semantic colors from our color system +- Pair icons with text labels when possible +- Use standard sizes (16px, 20px, 24px, 32px) +- Ensure sufficient contrast for accessibility + +### ❌ Don't + +- Mix different icon libraries or styles +- Modify the stroke width or visual style +- Use icons as pure decoration without function +- Use icons alone for complex or uncommon actions +- Scale icons to arbitrary sizes +- Create custom icons unless absolutely necessary + Icons should enhance usability and clarity, not complicate the interface. When in doubt, prioritize clear text labels over icons alone. + +# Layout + +Our layout system ensures **consistency, clarity, and usability** across all interfaces. These guidelines establish standardized patterns for organizing content and interface elements. + +## Form + +Forms are fundamental building blocks of our application. They should be clear, efficient, and follow consistent patterns to reduce cognitive load and improve user experience. + +### Design Principles + +- **Predictable structure**: Consistent vertical hierarchy helps users scan and complete forms efficiently +- **Clear communication**: Every element serves a purpose in guiding users toward successful completion +- **Minimal cognitive load**: Use established patterns and clear visual hierarchy +- **Accessible by default**: Follow semantic HTML and proper labeling conventions + +### Vertical Layout Guidelines + +All form elements follow a consistent top-to-bottom hierarchy: +**Label → Description → Input → Validation/Hint** +This predictable order allows users to quickly understand what information is needed and how to provide it correctly. + +### Spacing Guidelines + +Use consistent spacing to create clear relationships between form elements: + +- **4px gap (`gap-y-1`)** between all adjacent form elements: +- Label to Description +- Description to Input +- Input to Validation/Hint + This tight, consistent spacing groups related elements while maintaining clear separation between form fields. + +### Typography Guidelines + +Follow our established [typography system](../../visual_identity/3_typography/index.mdx) for form elements: + +#### Label + +- **Style**: Body emphasized (`text-xs font-semibold text-emphasis`) +- **Purpose**: Clearly identify what information is required +- **Example**: "Job name:", "Resource type:", "Environment variables:" + +#### Description + +- **Style**: Body (`text-xs font-normal text-secondary`) +- **Purpose**: Provide additional context or instructions +- **Example**: "Choose a descriptive name for your automation job" + +#### Validation/Hint + +- **Style**: Caption (`text-2xs font-normal text-hint`) +- **Purpose**: Guide users with requirements or feedback +- **Example**: "Required field", "Must be at least 8 characters", "Optional field" + +### Writing Guidelines + +#### Descriptions + +- Keep descriptions concise and actionable +- Focus on the outcome or benefit, not the technical process +- Use sentence case and avoid unnecessary punctuation +- Example: ✅ "Choose the Python version for your script execution" vs ❌ "This dropdown allows you to select which Python version will be used when executing your script." + +#### Helper Text and Hints + +- Be specific about requirements upfront +- Use positive language when possible +- Provide examples for complex inputs +- Example: ✅ "Use lowercase letters, numbers, and hyphens only" vs ❌ "Invalid characters not allowed" + +### Tooltip Usage + +Use tooltips sparingly for **additional context** that would otherwise clutter the interface: + +- Complex terminology or concepts that need definition +- Background information that helps with decision-making +- Links to relevant documentation or resources + **Don't use tooltips for**: +- Essential information needed to complete the form +- Error messages or validation feedback +- Basic instructions that should be in the description + +### Visual Example + +![Form layout pattern showing proper hierarchy and spacing (Light mode)](./static/brand-guidelines-assets/form-light.svg) +_Light mode example_ + +### Implementation Notes + +- Always include proper semantic HTML (`, `, etc.) +- Associate labels with inputs using `for` attributes or wrapping +- Maintain consistent spacing using Tailwind's `gap-y-1` utility +- Test form layouts across different viewport sizes +- Ensure sufficient color contrast for all text elements + +# Spacing & Layout + +Windmill uses **Tailwind CSS spacing utilities** and **flex containers** to create consistent, responsive layouts across all interfaces. + +## Spacing Scale + +Our spacing system follows **Tailwind's default spacing scale** based on 4px increments: + +- **`space-1`** (4px): Micro spacing between related elements +- **`space-2`** (8px): Base unit for component padding and margins +- **`space-4`** (16px): Standard spacing between components +- **`space-6`** (24px): Section spacing and larger gaps +- **`space-8`** (32px): Page margins and major sections +- **`space-12`** (48px): Large spacing for visual breaks +- **`space-16`** (64px): Maximum spacing for major layout divisions + +## Layout System + +We use **Tailwind's flex utilities** for responsive layouts: + +- **Flex containers**: `flex flex-col` or `flex flex-row` for layout direction +- **Spacing**: `space-x-4` and `space-y-4` for consistent gaps between elements +- **Max width**: `max-w-6xl` (1152px) for content areas + +## Do's and Don'ts + +### ✅ Do + +- Use Tailwind spacing utilities (`p-4`, `m-6`, `space-x-4`) for all measurements +- Use flex responsive classes (`flex-col md:flex-row`, `justify-between`) +- Apply consistent container patterns (`container mx-auto px-8`) +- Use `space-x-*` and `space-y-*` for gaps between flex items +- Leverage Tailwind's responsive breakpoints (`sm:`, `md:`, `lg:`, `xl:`) + +### ❌ Don't + +- Use arbitrary spacing values with square brackets `[32px]` +- Mix CSS spacing with Tailwind utilities in the same component +- Use fixed pixel values instead of responsive utilities +- Break responsive design patterns with custom CSS diff --git a/frontend/openapi-ts-error-1758271586180.log b/frontend/openapi-ts-error-1758271586180.log new file mode 100644 index 0000000000..4c0634a7c7 --- /dev/null +++ b/frontend/openapi-ts-error-1758271586180.log @@ -0,0 +1,28 @@ +Error parsing /home/rfiszel/windmill/backend/windmill-api/openapi.yaml: bad indentation of a mapping entry (8025:25) + + 8022 | description: job args + 8023 | content: + 8024 | application/json: + 8025 | schema: {}\ +--------------------------------^ + 8026 | + 8027 | /w/{workspace}/jobs/queue/get_scheduled_for_by_ids: +ParserError: Error parsing /home/rfiszel/windmill/backend/windmill-api/openapi.yaml: bad indentation of a mapping entry (8025:25) + + 8022 | description: job args + 8023 | content: + 8024 | application/json: + 8025 | schema: {}\ +--------------------------------^ + 8026 | + 8027 | /w/{workspace}/jobs/queue/get_scheduled_for_by_ids: + at Object.parse (/home/rfiszel/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/parsers/yaml.js:44:23) + at getResult (/home/rfiszel/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/util/plugins.js:116:22) + at runNextPlugin (/home/rfiszel/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/util/plugins.js:64:32) + at /home/rfiszel/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/util/plugins.js:55:9 + at new Promise () + at Object.run (/home/rfiszel/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/util/plugins.js:54:12) + at parseFile (/home/rfiszel/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/parse.js:130:38) + at parse (/home/rfiszel/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/parse.js:56:30) + at async $RefParser.parse (/home/rfiszel/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/index.js:115:28) + at async $RefParser.resolve (/home/rfiszel/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/index.js:145:13) \ No newline at end of file diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e0a185c5ef..56d162adee 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,24 +1,23 @@ { "name": "windmill-components", - "version": "1.542.1", + "version": "1.573.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.542.1", + "version": "1.573.3", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { "@anthropic-ai/sdk": "^0.60.0", "@aws-crypto/sha256-js": "^4.0.0", - "@codingame/monaco-vscode-configuration-service-override": "~20.2.1", - "@codingame/monaco-vscode-editor-api": "~20.2.1", - "@codingame/monaco-vscode-standalone-css-language-features": "~20.2.1", - "@codingame/monaco-vscode-standalone-html-language-features": "~20.2.1", - "@codingame/monaco-vscode-standalone-json-language-features": "~20.2.1", - "@codingame/monaco-vscode-standalone-languages": "~20.2.1", - "@codingame/monaco-vscode-standalone-typescript-language-features": "~20.2.1", + "@codingame/monaco-vscode-editor-api": "=21.6.0", + "@codingame/monaco-vscode-standalone-css-language-features": "=21.6.0", + "@codingame/monaco-vscode-standalone-html-language-features": "=21.6.0", + "@codingame/monaco-vscode-standalone-json-language-features": "=21.6.0", + "@codingame/monaco-vscode-standalone-languages": "=21.6.0", + "@codingame/monaco-vscode-standalone-typescript-language-features": "=21.6.0", "@json2csv/plainjs": "^7.0.6", "@leeoniya/ufuzzy": "^1.0.8", "@popperjs/core": "^2.11.6", @@ -51,10 +50,9 @@ "lru-cache": "^11.1.0", "lucide-svelte": "^0.540.0", "minimatch": "^10.0.1", - "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@~20.2.1", - "monaco-editor-wrapper": "6.12.0", + "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@=21.6.0", "monaco-graphql": "=1.6.0", - "monaco-languageclient": "9.11.0", + "monaco-languageclient": "10.1.0", "monaco-vim": "^0.4.1", "ol": "^7.4.0", "openai": "^5.16.0", @@ -62,6 +60,7 @@ "p-limit": "^6.1.0", "panzoom": "^9.4.3", "pdfjs-dist": "4.8.69", + "quicktype-core": "^23.2.6", "quill": "^1.3.7", "rehype-github-alerts": "^3.0.0", "rehype-raw": "^7.0.0", @@ -70,7 +69,7 @@ "svelte-exmarkdown": "^5.0.0", "svelte-infinite-loading": "^1.4.0", "tailwind-merge": "^1.13.2", - "vscode": "npm:@codingame/monaco-vscode-extension-api@~20.2.1", + "vscode": "npm:@codingame/monaco-vscode-extension-api@=21.6.0", "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.1.0", "vscode-ws-jsonrpc": "~3.5.0", @@ -80,13 +79,13 @@ "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.510.1", "windmill-parser-wasm-py": "1.538.0", - "windmill-parser-wasm-regex": "1.512.0", + "windmill-parser-wasm-regex": "1.565.0", "windmill-parser-wasm-ruby": "1.526.1", - "windmill-parser-wasm-rust": "1.510.1", - "windmill-parser-wasm-ts": "1.538.0", - "windmill-parser-wasm-yaml": "1.510.1", + "windmill-parser-wasm-rust": "1.558.1", + "windmill-parser-wasm-ts": "1.565.0", + "windmill-parser-wasm-yaml": "1.561.0", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.3.0", + "windmill-utils-internal": "^1.3.1", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", @@ -115,7 +114,6 @@ "@types/vscode": "^1.83.5", "@typescript-eslint/eslint-plugin": "^5.59.8", "@typescript-eslint/parser": "^5.60.0", - "@windmill-labs/esbuild-import-meta-url-plugin": "0.0.0-semantic-release", "@zerodevx/svelte-toast": "^0.9.6", "autoprefixer": "^10.4.13", "cssnano": "^6.0.1", @@ -144,7 +142,7 @@ "tar": "^7.4.3", "tslib": "^2.6.1", "typescript": "^5.5.0", - "vite": "^7.1.5", + "vite": "npm:rolldown-vite@latest", "vite-plugin-mkcert": "^1.17.5", "yootils": "^0.3.1" }, @@ -231,12 +229,12 @@ "license": "0BSD" }, "node_modules/@aws-sdk/types": { - "version": "3.862.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.862.0.tgz", - "integrity": "sha512-Bei+RL0cDxxV+lW2UezLbCYYNeJm6Nzee0TpW0FfyTRBhH9C1XQh4+x+IClriXvgBnRquTMMYsmJfvx8iyLKrg==", + "version": "3.901.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.901.0.tgz", + "integrity": "sha512-FfEM25hLEs4LoXsLXQ/q6X6L4JmKkKkbVFpKD4mwfVHtRVQG6QxJiCPcrkcPISquiy6esbwK2eh64TWbiD60cg==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.3.2", + "@smithy/types": "^4.6.0", "tslib": "^2.6.2" }, "engines": { @@ -288,1493 +286,1511 @@ "node": ">=6.9.0" } }, - "node_modules/@codingame/monaco-vscode-08d1b4da-daf2-5f0d-8c50-ca6a6986c50f-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-08d1b4da-daf2-5f0d-8c50-ca6a6986c50f-common/-/monaco-vscode-08d1b4da-daf2-5f0d-8c50-ca6a6986c50f-common-20.2.1.tgz", - "integrity": "sha512-2knRCAm0RMhRrBsQxpWPmCduNCoc03GqQs8Rkw6XupQYyZxNkgJPaoxdWrwZsmrx3FivCV6t/+ijl70X6dIRnQ==", + "node_modules/@codingame/monaco-vscode-05a2a821-e4de-5941-b7f9-bbf01c09f229-common": { + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-05a2a821-e4de-5941-b7f9-bbf01c09f229-common/-/monaco-vscode-05a2a821-e4de-5941-b7f9-bbf01c09f229-common-21.6.0.tgz", + "integrity": "sha512-K9eB11bREqotuq0ThCGJwp6JwndLkoZKw5z6dBjnCVDrPcUX5rZ7yV4891D1E6tBY0o8a9/Tvg5stgSzCqFVvA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-b6d52a6d-8c8e-51f5-bcd2-1722295e31d9-common": "21.6.0" + } + }, + "node_modules/@codingame/monaco-vscode-08d1b4da-daf2-5f0d-8c50-ca6a6986c50f-common": { + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-08d1b4da-daf2-5f0d-8c50-ca6a6986c50f-common/-/monaco-vscode-08d1b4da-daf2-5f0d-8c50-ca6a6986c50f-common-21.6.0.tgz", + "integrity": "sha512-FWwmf/tq+ApcbPJcFJ9ZzREEY24Fiy9kOVAgMW6IFAcJ5RejRYsO1I3dX4/aVckQYUBpaA65+Lfa7ArQVQ1WlQ==", + "license": "MIT", + "dependencies": { + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-0af61f78-dfc5-57ba-8d32-66268c8de38d-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-0af61f78-dfc5-57ba-8d32-66268c8de38d-common/-/monaco-vscode-0af61f78-dfc5-57ba-8d32-66268c8de38d-common-20.2.1.tgz", - "integrity": "sha512-6EvqHSjWuf5HhjHPv+CXurWq31QR+PrxnuWaRX3O+Qvse/hACIBlPXDAD8fDMn1YK9QSdf4rwds0zChbfpmosQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-0af61f78-dfc5-57ba-8d32-66268c8de38d-common/-/monaco-vscode-0af61f78-dfc5-57ba-8d32-66268c8de38d-common-21.6.0.tgz", + "integrity": "sha512-tBNQE4LAI5gAn1/vd0LaD+COX69Y5mWjp2SwBr19hmqusxbbZCVe6OGFrsolvBbW2fX8Hl8D8sP6+Daj8xS/kA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-b994942c-360d-5b68-8a33-77d4bde6b714-common": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-b994942c-360d-5b68-8a33-77d4bde6b714-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-0c06bfba-d24d-5c4d-90cd-b40cefb7f811-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-0c06bfba-d24d-5c4d-90cd-b40cefb7f811-common/-/monaco-vscode-0c06bfba-d24d-5c4d-90cd-b40cefb7f811-common-20.2.1.tgz", - "integrity": "sha512-PaB//D7uvOUay6TbPxt6GrVfRVQBF1JWe++3srdqAEn4vHS8yT3uS1GUI+ET0VPK4LNvkl5gEF04wC7y5fLXoQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-0c06bfba-d24d-5c4d-90cd-b40cefb7f811-common/-/monaco-vscode-0c06bfba-d24d-5c4d-90cd-b40cefb7f811-common-21.6.0.tgz", + "integrity": "sha512-VJgFxjh12XmGhEHwg0DjIgnLh0oS3kFFoz7gU/K9Bij8ERovIikdij+2C2z+/eG5IKynMJpl6Ga1yNa2txmvWw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-0cc5da60-f921-59b9-bd8c-a018e93c0a6f-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-0cc5da60-f921-59b9-bd8c-a018e93c0a6f-common/-/monaco-vscode-0cc5da60-f921-59b9-bd8c-a018e93c0a6f-common-20.2.1.tgz", - "integrity": "sha512-5Wk6XcM1BZyOopJ/XOea60FLNhx4ErrU4PYFwpb5NePlK6Bk27YPh1Bv3meohmRxr3WSdQymYGuRAdRj3stXoA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-0cc5da60-f921-59b9-bd8c-a018e93c0a6f-common/-/monaco-vscode-0cc5da60-f921-59b9-bd8c-a018e93c0a6f-common-21.6.0.tgz", + "integrity": "sha512-tX8pXT5GXPg+6X2VvuY7dbntenkSdI4+txk7B2zlIjX2dUe638CrZiF4tv6CICBFshS/DCWPPF85KZosMQ6wDw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-f22e7e55-aee8-5b52-a6bc-950efd9f5890-common": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-f22e7e55-aee8-5b52-a6bc-950efd9f5890-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-1021b67c-93e5-5c78-a270-cbdb2574d980-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-1021b67c-93e5-5c78-a270-cbdb2574d980-common/-/monaco-vscode-1021b67c-93e5-5c78-a270-cbdb2574d980-common-20.2.1.tgz", - "integrity": "sha512-jyH8bDlKbKnPBPE37c3UPhuE9QBlbSL//SW44vlbq64Lg8s/b58o87wMRhi2r2RvgzAgJhtTTn4WMNDAgFwGVg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-1021b67c-93e5-5c78-a270-cbdb2574d980-common/-/monaco-vscode-1021b67c-93e5-5c78-a270-cbdb2574d980-common-21.6.0.tgz", + "integrity": "sha512-zurQClxd0L5muvSSeR5WeX1Gee9NHWwOr1on77hET2WDko/+0jj2OVoJQtyh4Snskr5qjcdXgAwThz0l317A4A==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-15626ec7-b165-51e1-8caf-7bcc2ae9b95a-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-15626ec7-b165-51e1-8caf-7bcc2ae9b95a-common/-/monaco-vscode-15626ec7-b165-51e1-8caf-7bcc2ae9b95a-common-20.2.1.tgz", - "integrity": "sha512-CXw/aLqqMDism6lyFXuR+5r/AIgl5UgTXm0xli8WK38LoqhYtRMVxk5vy6kbmtK3w5cA9XMKwWSO5NHS8XscTQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-15626ec7-b165-51e1-8caf-7bcc2ae9b95a-common/-/monaco-vscode-15626ec7-b165-51e1-8caf-7bcc2ae9b95a-common-21.6.0.tgz", + "integrity": "sha512-Mig4Ts/8mkVyYG6PgrHbxVaIkX4Kw4eqnp7OXu5ZcYswbSc6gnLxSg58DOoJ0NsHX2BP+HxBj2XnBHQCS1Hgyg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common/-/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common-20.2.1.tgz", - "integrity": "sha512-WuzU3yRkKwL8ZXiZyEAF5xeWqEebJu3rWBndQYsig/12wbfeqEwQBh7JwZBK1uc/1Cl8VN8wE1QpYQyC/C7m8g==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common/-/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common-21.6.0.tgz", + "integrity": "sha512-UuW9A1hLvYA+rK1YGrXywJu7Dwh1Hl3IDKY9jguxhGHzQPoO3M6aX47q8bIMmo4Wd9B9W8TidxjcRjv3+FVxYA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-1b4486de-4fe4-59c4-9e6d-34f265ff6625-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-1b4486de-4fe4-59c4-9e6d-34f265ff6625-common/-/monaco-vscode-1b4486de-4fe4-59c4-9e6d-34f265ff6625-common-20.2.1.tgz", - "integrity": "sha512-ouRHL1DZqoBM9+57uES4tKu4ZGsCbaYXOj0MiDK9OTOMpEM0wNAIpOwT7mrVMrU+PtjhczgNJ6rr22l+ONKOoA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-1b4486de-4fe4-59c4-9e6d-34f265ff6625-common/-/monaco-vscode-1b4486de-4fe4-59c4-9e6d-34f265ff6625-common-21.6.0.tgz", + "integrity": "sha512-zjdFDkMxi7p6EtQabq0zm/up82eD8PcNO9JBzxsf02luy5Mwna4nkYAT8dQUfAR4nTu8UlJSiDSnpMOjFH6M8w==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-2a94c04a-b85b-5669-b06b-89c1bfa11cb9-common": "20.2.1", - "@codingame/monaco-vscode-422642f2-7e3a-5c1c-9e1e-1d3ef1817346-common": "20.2.1", - "@codingame/monaco-vscode-501b06ab-3f58-516b-8a1a-c29d375d3da4-common": "20.2.1", - "@codingame/monaco-vscode-9a1a5840-af83-5d07-a156-ba32a36c5c4b-common": "20.2.1", - "@codingame/monaco-vscode-a8d3bd74-e63e-5327-96e8-4f931661e329-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1" + "@codingame/monaco-vscode-2a94c04a-b85b-5669-b06b-89c1bfa11cb9-common": "21.6.0", + "@codingame/monaco-vscode-422642f2-7e3a-5c1c-9e1e-1d3ef1817346-common": "21.6.0", + "@codingame/monaco-vscode-4fad3647-b95d-5c19-bab1-bb9de627a5ec-common": "21.6.0", + "@codingame/monaco-vscode-71c8dbff-4c98-552f-aef0-e72b00fdcfc0-common": "21.6.0", + "@codingame/monaco-vscode-9a1a5840-af83-5d07-a156-ba32a36c5c4b-common": "21.6.0", + "@codingame/monaco-vscode-9a5ab9e7-d838-5831-9eb4-e79ea3764dcb-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-23aade48-f094-5c08-9555-97fc9cca96c9-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-23aade48-f094-5c08-9555-97fc9cca96c9-common/-/monaco-vscode-23aade48-f094-5c08-9555-97fc9cca96c9-common-20.2.1.tgz", - "integrity": "sha512-xV8T8zmmQhAb7NPz+0h6Z0R4IzdB3BcwXkYIKuANZLI2jeKdo+ixw2+FClCHGjGBycffmYqIISXecgixIJrJHw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-23aade48-f094-5c08-9555-97fc9cca96c9-common/-/monaco-vscode-23aade48-f094-5c08-9555-97fc9cca96c9-common-21.6.0.tgz", + "integrity": "sha512-Y9YH8avGz4wTV0dvI4u7qEBaZz3vY23HIUzx8iko7TCws2U+J4o7RXQcRcsKx1PORzfUIQTpI/1731nuA7F7Hw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" + } + }, + "node_modules/@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": { + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common/-/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common-21.6.0.tgz", + "integrity": "sha512-kMYJTMDWugg+JHIFYr3KaSMv1OlJSuhpom0Y+BlLZr7jnyaN/k55PwqfeuJmX5a18e3hwu3PX9abM2/dBk27Kw==", + "license": "MIT", + "dependencies": { + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-249dc928-1da3-51c1-82d0-45e0ba9d08a1-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-249dc928-1da3-51c1-82d0-45e0ba9d08a1-common/-/monaco-vscode-249dc928-1da3-51c1-82d0-45e0ba9d08a1-common-20.2.1.tgz", - "integrity": "sha512-3MvmIwes9tLQd7FopLUtsocjRiej+dxbp+Eo5uQrPyV4sghyrhuSpe90mHM7MK3Pd/YNmtRXAem5EtRkWeyCog==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-249dc928-1da3-51c1-82d0-45e0ba9d08a1-common/-/monaco-vscode-249dc928-1da3-51c1-82d0-45e0ba9d08a1-common-21.6.0.tgz", + "integrity": "sha512-c/peZ/OGO4nUz9F052zTIZ2E6VplCre+aZAX3EDKbIiNVGirWvWUT2JnORdxjPb1WAiWS++5usWnpe/YHKSxBA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-256d5b78-0649-50e9-8354-2807f95f68f4-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-256d5b78-0649-50e9-8354-2807f95f68f4-common/-/monaco-vscode-256d5b78-0649-50e9-8354-2807f95f68f4-common-20.2.1.tgz", - "integrity": "sha512-692CODk83dErjqKqdQKu87eKVV10lXXRNOS5tbbd3FqIdDiEYubocKDkTXSwZ5wVfQYmALaUEsfSN4XIqUVLAA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-256d5b78-0649-50e9-8354-2807f95f68f4-common/-/monaco-vscode-256d5b78-0649-50e9-8354-2807f95f68f4-common-21.6.0.tgz", + "integrity": "sha512-f/uy2BH/88r7eh/0VjMEbgos+28tqEK1Mb/jwRRNSpAq+W4goca+qiN0gXxwvqMlzOYbL25Q1diqfLBuPWpFWw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" - } - }, - "node_modules/@codingame/monaco-vscode-262ed59d-4f76-57cd-9e9f-1877f26ae049-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-262ed59d-4f76-57cd-9e9f-1877f26ae049-common/-/monaco-vscode-262ed59d-4f76-57cd-9e9f-1877f26ae049-common-20.2.1.tgz", - "integrity": "sha512-MHK36sZwljQ+U3/wDrY57sN/iN6tM5+Xu4uBZpdnEorkoXq/Jg2q8dvVXw0qGvHfR80oT5i/dBdpcdytrR9ADA==", - "license": "MIT", - "dependencies": { - "@codingame/monaco-vscode-6883db80-c313-54eb-8fbc-5872c56b0326-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-2a22c7b4-b906-5914-8cd1-3ed912fb738f-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-2a22c7b4-b906-5914-8cd1-3ed912fb738f-common/-/monaco-vscode-2a22c7b4-b906-5914-8cd1-3ed912fb738f-common-20.2.1.tgz", - "integrity": "sha512-c8cONu9sQ2xhpYtyKDxMWAbk+yqk0VLulueUSbHYQk4AAiOaMb0D27pf3A1rVMkXhwma8cLvozLlMFMWcWUnbQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-2a22c7b4-b906-5914-8cd1-3ed912fb738f-common/-/monaco-vscode-2a22c7b4-b906-5914-8cd1-3ed912fb738f-common-21.6.0.tgz", + "integrity": "sha512-r3R1ciw2D7hWb+tVi7LHq6AeTKrpYXaEpJOtC4fr2Mr+bzECvaTs6KvaDLXd7GEpiW61Lbxz5JHWj4J9vQS4WQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-2a94c04a-b85b-5669-b06b-89c1bfa11cb9-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-2a94c04a-b85b-5669-b06b-89c1bfa11cb9-common/-/monaco-vscode-2a94c04a-b85b-5669-b06b-89c1bfa11cb9-common-20.2.1.tgz", - "integrity": "sha512-R75FXTJR5mO1u57E3a5jBnEfxeIAPmRVEp5i+w9wGh/IkvQ/VidK1eH1g1lK/m2CRGcgzY6pYwMb7bbrOqzNHg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-2a94c04a-b85b-5669-b06b-89c1bfa11cb9-common/-/monaco-vscode-2a94c04a-b85b-5669-b06b-89c1bfa11cb9-common-21.6.0.tgz", + "integrity": "sha512-xaiRmKLOO7CwuQQff/Pe6ObQm/+vPr8ignXspqNGvo47yQqkTChJlEa6P3jznIbOreOz/kVR7NS5a56a8sY5Rg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-4a3ac544-9a61-534c-88df-756262793ef7-common": "20.2.1", - "@codingame/monaco-vscode-a8d3bd74-e63e-5327-96e8-4f931661e329-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-4a3ac544-9a61-534c-88df-756262793ef7-common": "21.6.0", + "@codingame/monaco-vscode-71c8dbff-4c98-552f-aef0-e72b00fdcfc0-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-2f06fe84-148e-5e6b-a7ca-c7989c5f128a-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-2f06fe84-148e-5e6b-a7ca-c7989c5f128a-common/-/monaco-vscode-2f06fe84-148e-5e6b-a7ca-c7989c5f128a-common-20.2.1.tgz", - "integrity": "sha512-Gz5ukIwb/g+8KMhlAuPK2DaL5MkHJRpBk/jjsTGu7E1AvJWcxFMxk4Z0P3KhhZCwZxbAH2OmP3u6dAtHOGTc2Q==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-2f06fe84-148e-5e6b-a7ca-c7989c5f128a-common/-/monaco-vscode-2f06fe84-148e-5e6b-a7ca-c7989c5f128a-common-21.6.0.tgz", + "integrity": "sha512-Wg4YeC5pVag97B5BD0tl3016LJ4kHjhnygJo1PmD4yU36q7l7DXLa10tH38aDfcEOyYtEfZsssq2JjtqQOpgig==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-3109a756-1f83-5d09-945b-9f0fcad928f0-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-3109a756-1f83-5d09-945b-9f0fcad928f0-common/-/monaco-vscode-3109a756-1f83-5d09-945b-9f0fcad928f0-common-20.2.1.tgz", - "integrity": "sha512-fNyMykAebJsUifUrCk542lO78tE4+SoneZQD8+ULcnq8hf8lXkFJ64vXB/afJyNHgX3BtTI0LtPAkam8ClNLvg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-3109a756-1f83-5d09-945b-9f0fcad928f0-common/-/monaco-vscode-3109a756-1f83-5d09-945b-9f0fcad928f0-common-21.6.0.tgz", + "integrity": "sha512-lqV4EnQ4kOoahNYtbpUUhonyAIZa+yStx9qYgiHzYJQ/0v7Zy8oR1CFTG9lBRpUy0fONTXHnhUbqeSDQv3tTSw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-ac93482b-2178-52df-a200-ba0d1a4963fb-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-bc6d9a89-1625-5010-b57e-ff44151144fe-common": "20.2.1" + "@codingame/monaco-vscode-ac93482b-2178-52df-a200-ba0d1a4963fb-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-bc6d9a89-1625-5010-b57e-ff44151144fe-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-33833ac7-3af3-5e9d-8fb9-11838d852c59-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-33833ac7-3af3-5e9d-8fb9-11838d852c59-common/-/monaco-vscode-33833ac7-3af3-5e9d-8fb9-11838d852c59-common-20.2.1.tgz", - "integrity": "sha512-MfY2NuHZQxFTVfoJte6377KkgrBfXbV1xZqczFG8BEs9ecJmNRMtoYe6fsXkvZIqUuSdGLLazr1OrKFMCinhEg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-33833ac7-3af3-5e9d-8fb9-11838d852c59-common/-/monaco-vscode-33833ac7-3af3-5e9d-8fb9-11838d852c59-common-21.6.0.tgz", + "integrity": "sha512-GYBi3j4qHN79GS3ErRGgkaTUC4b0ecGBDsGSvMbErQpCvuaDMZEee78DfVzjYPJG1Xzlp1wA9UBsIn15+x0gfw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" - } - }, - "node_modules/@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common/-/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common-20.2.1.tgz", - "integrity": "sha512-y37l/C9r8jkWLRZDAiVYSK8kZdFjMvN0cxLrRbOL/O/TSKCa7sdT7fnmelwwSwY/moFVknrSUSJf1HLiZeB5yw==", - "license": "MIT", - "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-40cada32-7e9c-528a-81fc-766e4da54147-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-40cada32-7e9c-528a-81fc-766e4da54147-common/-/monaco-vscode-40cada32-7e9c-528a-81fc-766e4da54147-common-20.2.1.tgz", - "integrity": "sha512-wCxhPyTty6XWwgIl492WUCXI5Zuwxrhc+BgZPjm32GCk2D/MZMhJt/T7iH8ZkCIGtO1kIyHpBXDBJL7L2h2lag==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-40cada32-7e9c-528a-81fc-766e4da54147-common/-/monaco-vscode-40cada32-7e9c-528a-81fc-766e4da54147-common-21.6.0.tgz", + "integrity": "sha512-2g7cvM9VrfWCnza//1ytdj+LvbkaRLjsXRwDCbY9yPZkIg6/IJmfS3JM45tllZaLUXs2yPCg9zhvaVCBfcijLg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "20.2.1", - "@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "21.6.0", + "@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-422642f2-7e3a-5c1c-9e1e-1d3ef1817346-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-422642f2-7e3a-5c1c-9e1e-1d3ef1817346-common/-/monaco-vscode-422642f2-7e3a-5c1c-9e1e-1d3ef1817346-common-20.2.1.tgz", - "integrity": "sha512-KiPs0Bz2NdBUbfN+SrbzAXmCK1n1sk4C/EBHcjAU9EFt+9Qs7vBWPbPL/a9uxsAn1yr3/ENKRA/sG9la3TJP6g==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-422642f2-7e3a-5c1c-9e1e-1d3ef1817346-common/-/monaco-vscode-422642f2-7e3a-5c1c-9e1e-1d3ef1817346-common-21.6.0.tgz", + "integrity": "sha512-GavXhLlNHfAOsRQWS6oP//HuFd2UQ8LkXr1zPAwUWxiGwVyQ3zMEju4KhnipU1ex4ryKEmOWmClpZah2UfGT2A==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-494be54c-bd37-5b3c-af70-02f086e28768-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-494be54c-bd37-5b3c-af70-02f086e28768-common/-/monaco-vscode-494be54c-bd37-5b3c-af70-02f086e28768-common-20.2.1.tgz", - "integrity": "sha512-UKLwoBLElU8PHVGdqkbPxMzPNA6RD2C7qHNFkjff0tqxMPifLnVPfXzhHuGfFhUuWU4Oo77MZ6kB9swJ/Zd3iQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-494be54c-bd37-5b3c-af70-02f086e28768-common/-/monaco-vscode-494be54c-bd37-5b3c-af70-02f086e28768-common-21.6.0.tgz", + "integrity": "sha512-Y0jwYhs4/RfRjM4L21+t5e/4CEhEDYnMR4mcxfsMq3c+sWJUIj1ExYU5P7L8Pcv9gd8BFTym7tdT07Ls0C0Vcg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-4a316137-39d1-5d77-8b53-112db3547c1e-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-4a316137-39d1-5d77-8b53-112db3547c1e-common/-/monaco-vscode-4a316137-39d1-5d77-8b53-112db3547c1e-common-20.2.1.tgz", - "integrity": "sha512-DKCktdRnY9QUZiG3aZHMReGiWpJKzXdbTWC1sffxMKyRTlWp+jjy6VD5LZKNSEIjks92emq02JNW6B4HQqdNuw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-4a316137-39d1-5d77-8b53-112db3547c1e-common/-/monaco-vscode-4a316137-39d1-5d77-8b53-112db3547c1e-common-21.6.0.tgz", + "integrity": "sha512-KLZdyUCUr1ZPE25Od8bCraq7nzFdIBvSta+uGfGpT8qGI+GuU5wr0Xn7Kk4tDhU4lYvoFJzsHssQf+90ICqWJQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-501b06ab-3f58-516b-8a1a-c29d375d3da4-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1", - "@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-4fad3647-b95d-5c19-bab1-bb9de627a5ec-common": "21.6.0", + "@codingame/monaco-vscode-9a5ab9e7-d838-5831-9eb4-e79ea3764dcb-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-4a3ac544-9a61-534c-88df-756262793ef7-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-4a3ac544-9a61-534c-88df-756262793ef7-common/-/monaco-vscode-4a3ac544-9a61-534c-88df-756262793ef7-common-20.2.1.tgz", - "integrity": "sha512-vjDxz1Vtnp7SXuh9IijlLyTH9Kz2gK432UiFGea3CyL9JRh6lF5MGl7r0qUeXQoyPoyTgvcUvhARnbSqtt3esw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-4a3ac544-9a61-534c-88df-756262793ef7-common/-/monaco-vscode-4a3ac544-9a61-534c-88df-756262793ef7-common-21.6.0.tgz", + "integrity": "sha512-f5E4lm+aEJ8Sz8nc6/OhEJRRPW1hMTES0xeaYm448VKwfxbEqyT4VMMF3ZpEE+rUux05CyDqMxAOGAhbT6rWgw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, - "node_modules/@codingame/monaco-vscode-501b06ab-3f58-516b-8a1a-c29d375d3da4-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-501b06ab-3f58-516b-8a1a-c29d375d3da4-common/-/monaco-vscode-501b06ab-3f58-516b-8a1a-c29d375d3da4-common-20.2.1.tgz", - "integrity": "sha512-5py+dgP5LT1LcK4mlWIc7GByW4Ft4u5MA0BimFTSAWY+B+32TlzUulXETLa3fdTBjthpPxZcfUivwKCIURe+Lw==", + "node_modules/@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common": { + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common/-/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common-21.6.0.tgz", + "integrity": "sha512-kmALyhBnH1umkJyxj6nb1rL/sJEp4fDUh7GFK/VmJgBsAw6dDXuZxGlZtO4qSvigJz5jcIcBMvcctqX7G1HjdA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" + } + }, + "node_modules/@codingame/monaco-vscode-4dda7789-5a25-5e8b-b2de-c2f11b1b96e5-common": { + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-4dda7789-5a25-5e8b-b2de-c2f11b1b96e5-common/-/monaco-vscode-4dda7789-5a25-5e8b-b2de-c2f11b1b96e5-common-21.6.0.tgz", + "integrity": "sha512-9cp7YtFkhCaMs/Ge66dJGb1MNGQhaN5Rd9MrSPcoNjL8jtCpiwYdlzssc8WCWLNTZpI7A2pQDpl+DYHuCpagzQ==", + "license": "MIT", + "dependencies": { + "@codingame/monaco-vscode-api": "21.6.0" + } + }, + "node_modules/@codingame/monaco-vscode-4fad3647-b95d-5c19-bab1-bb9de627a5ec-common": { + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-4fad3647-b95d-5c19-bab1-bb9de627a5ec-common/-/monaco-vscode-4fad3647-b95d-5c19-bab1-bb9de627a5ec-common-21.6.0.tgz", + "integrity": "sha512-KslhPUJ+lQ+BJzQkjfuPF3aRk1QVAiqPBlKwjCxPAkGa8QZu4PaXeeCFlFAnkVZ2qCEn/bcgZ1Q0VfS9QxWrSw==", + "license": "MIT", + "dependencies": { + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-523730aa-81e6-55d7-9916-87ad537fe087-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-523730aa-81e6-55d7-9916-87ad537fe087-common/-/monaco-vscode-523730aa-81e6-55d7-9916-87ad537fe087-common-20.2.1.tgz", - "integrity": "sha512-MO2j1DkLn3z8X8MYKfqSI1XrJBZaOABMLc/eg2PNhxpdSahqIzCNsenuW4BecUKn8VyG6W3GOOATJKaNe9Lmug==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-523730aa-81e6-55d7-9916-87ad537fe087-common/-/monaco-vscode-523730aa-81e6-55d7-9916-87ad537fe087-common-21.6.0.tgz", + "integrity": "sha512-U9YZBjLC9EG5ysrBtjCJnKH2r/H7QxDOVZKjmrPBDABB7k+NBxkypwrbpr5wQg1i9Sz6MSa43kUs5M94JQ57ag==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-a17e9d37-b6c1-5556-8402-5db73960fae3-common": "20.2.1", - "@codingame/monaco-vscode-ac93482b-2178-52df-a200-ba0d1a4963fb-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1" + "@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common": "21.6.0", + "@codingame/monaco-vscode-ac93482b-2178-52df-a200-ba0d1a4963fb-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-5452e2b7-9081-5f95-839b-4ab3544ce28f-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-5452e2b7-9081-5f95-839b-4ab3544ce28f-common/-/monaco-vscode-5452e2b7-9081-5f95-839b-4ab3544ce28f-common-20.2.1.tgz", - "integrity": "sha512-Doc26S+nDEsDT1CoLs4nB1wkEtg35HLZmig7xzObtw8b35pBXE9FmVe8aL/sowmSIb1fVflszunp8AyZGcyLpg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-5452e2b7-9081-5f95-839b-4ab3544ce28f-common/-/monaco-vscode-5452e2b7-9081-5f95-839b-4ab3544ce28f-common-21.6.0.tgz", + "integrity": "sha512-708ArbbT9pRAUf4UH5y6V6pk2PW0pWq1wXKYcTF3FqzD3juATnfMqyOrKxeNkJxmT3eY8LgpntF7vhmGjhfuMw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-0af61f78-dfc5-57ba-8d32-66268c8de38d-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-0af61f78-dfc5-57ba-8d32-66268c8de38d-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-571c8352-7953-5038-9f09-e03bb6219a0e-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-571c8352-7953-5038-9f09-e03bb6219a0e-common/-/monaco-vscode-571c8352-7953-5038-9f09-e03bb6219a0e-common-20.2.1.tgz", - "integrity": "sha512-wnGmFRFs9LRMDkfuXhLgg/WnsMpI25N5br8eAj1+RiYv9LdbWhQZjXHY0Bba3HNrVGEvH7SxGZ3tBeH5mH77MQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-571c8352-7953-5038-9f09-e03bb6219a0e-common/-/monaco-vscode-571c8352-7953-5038-9f09-e03bb6219a0e-common-21.6.0.tgz", + "integrity": "sha512-0QafpW7DvCSHMWarscbXB7jfIQJJU3bBhN9MYgTiCH4eTtfBI6/5ttRH0UkamxE7o/iwesEohXzVvhT+q9yVKw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-249dc928-1da3-51c1-82d0-45e0ba9d08a1-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, - "node_modules/@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common/-/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common-20.2.1.tgz", - "integrity": "sha512-6JLv8gpbbHsqg91ImVt21OXcr8w+THi6yAzgzrlZo1ok9nU5ARY7XjiBCJTajleyMj9tLnn+LbAkqE7A3I7BnA==", + "node_modules/@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common": { + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common/-/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common-21.6.0.tgz", + "integrity": "sha512-LmjgMsWG5qDbXNdJWcUmXZWZMpVrmyoucorLspFxRP9QOJmoKKAGCmvzDwKMQnmCeAXgGGdbvdPNKKgYVVs73A==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-615ce609-8555-545a-a549-47bd9f80e9f8-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-615ce609-8555-545a-a549-47bd9f80e9f8-common/-/monaco-vscode-615ce609-8555-545a-a549-47bd9f80e9f8-common-20.2.1.tgz", - "integrity": "sha512-PKvV+6yFq4zDXkqT8O5r/yu4quJsHECh0bRDp0SyJby3xayjyjVleY4Dp3KEgcifmAmBiaevY8+Qsoh8AqGa9A==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-615ce609-8555-545a-a549-47bd9f80e9f8-common/-/monaco-vscode-615ce609-8555-545a-a549-47bd9f80e9f8-common-21.6.0.tgz", + "integrity": "sha512-Wh0yD3YjsHuemYIdMJ7Oz1lFlzUbmCs0x69LMpoFRucZ/+hsYpig/9lSeF5uXCoRN8pz3gglgC2Xj12abgkHJA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-08d1b4da-daf2-5f0d-8c50-ca6a6986c50f-common": "20.2.1", - "@codingame/monaco-vscode-501b06ab-3f58-516b-8a1a-c29d375d3da4-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-622c0cca-d5fa-59b6-b730-0715afcf93ee-common": "20.2.1", - "@codingame/monaco-vscode-a17e9d37-b6c1-5556-8402-5db73960fae3-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1", - "@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common": "20.2.1" + "@codingame/monaco-vscode-08d1b4da-daf2-5f0d-8c50-ca6a6986c50f-common": "21.6.0", + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-4fad3647-b95d-5c19-bab1-bb9de627a5ec-common": "21.6.0", + "@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common": "21.6.0", + "@codingame/monaco-vscode-622c0cca-d5fa-59b6-b730-0715afcf93ee-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0", + "@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-622c0cca-d5fa-59b6-b730-0715afcf93ee-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-622c0cca-d5fa-59b6-b730-0715afcf93ee-common/-/monaco-vscode-622c0cca-d5fa-59b6-b730-0715afcf93ee-common-20.2.1.tgz", - "integrity": "sha512-7CJTGnACYkU4eZZz6ZgEUSacF1uVxgQp5PB/dDvTwjUHGSAoYDuYcUwGlRHiglhTM3OeWfr3bD1v93TM8t4tvQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-622c0cca-d5fa-59b6-b730-0715afcf93ee-common/-/monaco-vscode-622c0cca-d5fa-59b6-b730-0715afcf93ee-common-21.6.0.tgz", + "integrity": "sha512-Fw0feOxV04smlz7MrL+q/bF41ZKXukclYwLcySUkbg022wv4EWpWzdfGMSI7We4N78CUJNo4rVVz+KrmguycGA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common/-/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common-20.2.1.tgz", - "integrity": "sha512-ju1ZXDXt7HRiXUQ56xgqno6aIBqT8DjCbGq4lEpbV4LCa0cEUTN1vnPvfv6h5XgVfxxV7AgG/0223aJH/FvCcQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common/-/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common-21.6.0.tgz", + "integrity": "sha512-56sk3aYN56E52TLSpLOx3jKVbEvrVCB7e6I2FNGWlSRZGm4WQAtd8EJjfZ5CFryhpQc3DnNI6MrXc6jyIblkOQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-6845754f-e617-5ed9-8aaa-6ca3653a9532-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-6845754f-e617-5ed9-8aaa-6ca3653a9532-common/-/monaco-vscode-6845754f-e617-5ed9-8aaa-6ca3653a9532-common-20.2.1.tgz", - "integrity": "sha512-4hmesrfHj/NSJg7mPpN8OxrQtOjI4DW1nU8IqLgxjcYzUG0lKtBbyAdYEnrvace70+1QpeufTf3QZBgnLemI7A==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-6845754f-e617-5ed9-8aaa-6ca3653a9532-common/-/monaco-vscode-6845754f-e617-5ed9-8aaa-6ca3653a9532-common-21.6.0.tgz", + "integrity": "sha512-6kiB5YFftQXfRz5A9Cwl0knNCa2o9EIh1anKrYamOqgFEZSI1wxNl4t539ne2OtERYNHOoZ7jNxbuhKeNXpO7w==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" - } - }, - "node_modules/@codingame/monaco-vscode-6883db80-c313-54eb-8fbc-5872c56b0326-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-6883db80-c313-54eb-8fbc-5872c56b0326-common/-/monaco-vscode-6883db80-c313-54eb-8fbc-5872c56b0326-common-20.2.1.tgz", - "integrity": "sha512-LbKvvmNoE1outKNVNLZrlNxkmXbiX0ZmKc3epvDH/JsJMKzgP8BwObOuemwp0BjsAR2NVTtG0RPRUMFFACfJaA==", - "license": "MIT", - "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-6980eeab-47bb-5a48-8e15-32caf0785565-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-6980eeab-47bb-5a48-8e15-32caf0785565-common/-/monaco-vscode-6980eeab-47bb-5a48-8e15-32caf0785565-common-20.2.1.tgz", - "integrity": "sha512-nwlWlz5XeHhZ5bHBrBbl4Xoh7TkaI2kZgBsTaAY2CsQU8kyJ4qT9vkprTaThPWevzx3QWXbQcxs9MtHl9Sk2qw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-6980eeab-47bb-5a48-8e15-32caf0785565-common/-/monaco-vscode-6980eeab-47bb-5a48-8e15-32caf0785565-common-21.6.0.tgz", + "integrity": "sha512-iLffAYdvgWn8Uy9NpCNbisPn+i0Rhdq1ULVOkUQqvmqhvOSjnPN/9PGPGgScDLLLROZpJ/QYzxWEVhdvUd4S5w==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-0cc5da60-f921-59b9-bd8c-a018e93c0a6f-common": "20.2.1", - "@codingame/monaco-vscode-40cada32-7e9c-528a-81fc-766e4da54147-common": "20.2.1", - "@codingame/monaco-vscode-501b06ab-3f58-516b-8a1a-c29d375d3da4-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-615ce609-8555-545a-a549-47bd9f80e9f8-common": "20.2.1", - "@codingame/monaco-vscode-622c0cca-d5fa-59b6-b730-0715afcf93ee-common": "20.2.1", - "@codingame/monaco-vscode-6bf85d7b-e6e3-54e9-9bc1-7e08d663f0f6-common": "20.2.1", - "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "20.2.1", - "@codingame/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common": "20.2.1", - "@codingame/monaco-vscode-a8d3bd74-e63e-5327-96e8-4f931661e329-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-0cc5da60-f921-59b9-bd8c-a018e93c0a6f-common": "21.6.0", + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-40cada32-7e9c-528a-81fc-766e4da54147-common": "21.6.0", + "@codingame/monaco-vscode-4fad3647-b95d-5c19-bab1-bb9de627a5ec-common": "21.6.0", + "@codingame/monaco-vscode-615ce609-8555-545a-a549-47bd9f80e9f8-common": "21.6.0", + "@codingame/monaco-vscode-622c0cca-d5fa-59b6-b730-0715afcf93ee-common": "21.6.0", + "@codingame/monaco-vscode-6bf85d7b-e6e3-54e9-9bc1-7e08d663f0f6-common": "21.6.0", + "@codingame/monaco-vscode-71c8dbff-4c98-552f-aef0-e72b00fdcfc0-common": "21.6.0", + "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "21.6.0", + "@codingame/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-6bf85d7b-e6e3-54e9-9bc1-7e08d663f0f6-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-6bf85d7b-e6e3-54e9-9bc1-7e08d663f0f6-common/-/monaco-vscode-6bf85d7b-e6e3-54e9-9bc1-7e08d663f0f6-common-20.2.1.tgz", - "integrity": "sha512-+l9owwUpqkQgA9g0Mo23uQL2YoiMLEAZiR/rR4ti9T2OjRoejeH3FUjiutl+6Z7vq7WfSn2fl9DIqd6oC/BX8w==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-6bf85d7b-e6e3-54e9-9bc1-7e08d663f0f6-common/-/monaco-vscode-6bf85d7b-e6e3-54e9-9bc1-7e08d663f0f6-common-21.6.0.tgz", + "integrity": "sha512-TXW8DYt9eNq2hhgaws6iF4JBBx8i++5Qby3r3uLegokSYaRt29DWAaTITvkumO82Yh8S5fn9jic9At+cgme8qQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-0cc5da60-f921-59b9-bd8c-a018e93c0a6f-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "20.2.1", - "@codingame/monaco-vscode-96e83782-7f38-572e-8787-02e981f1c54f-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-f22e7e55-aee8-5b52-a6bc-950efd9f5890-common": "20.2.1" + "@codingame/monaco-vscode-0cc5da60-f921-59b9-bd8c-a018e93c0a6f-common": "21.6.0", + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "21.6.0", + "@codingame/monaco-vscode-96e83782-7f38-572e-8787-02e981f1c54f-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-f22e7e55-aee8-5b52-a6bc-950efd9f5890-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-6db1b967-5327-5c5c-8c17-bd92774c0fb2-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-6db1b967-5327-5c5c-8c17-bd92774c0fb2-common/-/monaco-vscode-6db1b967-5327-5c5c-8c17-bd92774c0fb2-common-20.2.1.tgz", - "integrity": "sha512-zYgzAYrrOgffttoGEG37f5VI4ZwpycZcoRA0wCPckk0rlTYAiHA4wabm8aR3H+IriWmAxBYqFyIg3b9EnsWWgA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-6db1b967-5327-5c5c-8c17-bd92774c0fb2-common/-/monaco-vscode-6db1b967-5327-5c5c-8c17-bd92774c0fb2-common-21.6.0.tgz", + "integrity": "sha512-SNXAKiAMUZeIL998yaMT+2MBy4SyiqyQBw892vPYtLu2XoaXPzhlpETPLHAIk1huVr3RVpW2tPYPUkyUkU7b6w==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, - "node_modules/@codingame/monaco-vscode-6f931a91-88ea-5232-897f-a17ec3929ba5-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-6f931a91-88ea-5232-897f-a17ec3929ba5-common/-/monaco-vscode-6f931a91-88ea-5232-897f-a17ec3929ba5-common-20.2.1.tgz", - "integrity": "sha512-5yQtd1/gHEUXrpvuZdGGBkJu4qesMspG/EyxPXEPWgWA03VsU+ZOv+J+pY6z9lQSKLrd0vMoBrZ+6Klg5TFQVw==", + "node_modules/@codingame/monaco-vscode-71c8dbff-4c98-552f-aef0-e72b00fdcfc0-common": { + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-71c8dbff-4c98-552f-aef0-e72b00fdcfc0-common/-/monaco-vscode-71c8dbff-4c98-552f-aef0-e72b00fdcfc0-common-21.6.0.tgz", + "integrity": "sha512-+ncY+YSUOu+0dPXBlFnebLfqoBNNbVhmfX7j/CGZyn66cmsqb5CNSEtEe/cUPsYE7lyKr/adJtO0UJjbWH/7xA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-262ed59d-4f76-57cd-9e9f-1877f26ae049-common": "20.2.1", - "@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-4fad3647-b95d-5c19-bab1-bb9de627a5ec-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-72a1b7d3-3f58-5545-9b7e-f579bd003081-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-72a1b7d3-3f58-5545-9b7e-f579bd003081-common/-/monaco-vscode-72a1b7d3-3f58-5545-9b7e-f579bd003081-common-20.2.1.tgz", - "integrity": "sha512-h12tZ7Q70p4YUcA3eEMxeg7L3SIoXC0Z6D4JqS+4N5FoXALKzk5vUw+RQBmpCpKisliwfMqBjoM/kj0dU56+GA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-72a1b7d3-3f58-5545-9b7e-f579bd003081-common/-/monaco-vscode-72a1b7d3-3f58-5545-9b7e-f579bd003081-common-21.6.0.tgz", + "integrity": "sha512-nPCXVFFciJSdceyswWcg7bmE7wmMTg81tkEZ2P63ZyeyCrombxJqj9gRHMdRaffr705dbDT75nvW84PeKImVwg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-a17e9d37-b6c1-5556-8402-5db73960fae3-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1", - "@codingame/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common": "20.2.1" + "@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0", + "@codingame/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-7869cfe8-f42c-5721-9f2b-7d04a6a41f16-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-7869cfe8-f42c-5721-9f2b-7d04a6a41f16-common/-/monaco-vscode-7869cfe8-f42c-5721-9f2b-7d04a6a41f16-common-20.2.1.tgz", - "integrity": "sha512-NN7jeVOULA9pXTCU7Bs98t1jZIvq+QIaUUWuuMq3CBUl8/H8T7ffcE3iQTv6b0x18VjjYydfw9JVfdcRkcEAig==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-7869cfe8-f42c-5721-9f2b-7d04a6a41f16-common/-/monaco-vscode-7869cfe8-f42c-5721-9f2b-7d04a6a41f16-common-21.6.0.tgz", + "integrity": "sha512-JL9zIiW5suPr4wrnmpU/2Lbyd4WewRkvBidBvb3m2KcZHRmF0K88Q/5cci5wNz/t21L97nqBw00hMj+6hbN96g==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-7f39b6f1-3542-5430-8760-0f404d7a7cee-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-7f39b6f1-3542-5430-8760-0f404d7a7cee-common/-/monaco-vscode-7f39b6f1-3542-5430-8760-0f404d7a7cee-common-20.2.1.tgz", - "integrity": "sha512-1M19sJ7ied1QvQN8V0zyPu1Rt0P2vzCGfDxsBhIMPsZpwUXDZ1k2P+HFQBjVshTlh0eQEvZNXWLs9yiuYl8I+g==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-7f39b6f1-3542-5430-8760-0f404d7a7cee-common/-/monaco-vscode-7f39b6f1-3542-5430-8760-0f404d7a7cee-common-21.6.0.tgz", + "integrity": "sha512-gNW6Dmb5QkJ35sqWWG0JcrPgRJFio8sJQ2czPBB8GEprUwvbRYUMq8ybnmJKdBmY4K2a4qny6guzrMe8MowLdg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common": "21.6.0", + "@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common/-/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common-20.2.1.tgz", - "integrity": "sha512-BjMRdAfVn3zb5l7RnlrgxSmSwCRFjeEYZQYbwa0EV6Jp0PBe2DFaOf2HZUrtVoLC6UlyC95DcDNcrmX8qdVAwA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common/-/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common-21.6.0.tgz", + "integrity": "sha512-E27CGVwWPH14U9fggCg2e4pRoFaZ2ae/kOXGz/R0qkGhWuc/hJsq8DGdD5+Eu7N0Bq/E0jnizEy/0/97sloQAA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-897bebad-39df-57cb-8a57-36a271d038be-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-897bebad-39df-57cb-8a57-36a271d038be-common/-/monaco-vscode-897bebad-39df-57cb-8a57-36a271d038be-common-20.2.1.tgz", - "integrity": "sha512-mYMsqVYVTLmoX9TYccX5QmEmcPGAuhnHSYu7KY7uNk/wPVeMmg/FQLyAMfXwPbmeSOcGlgD+qRLLBeapefVnFg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-897bebad-39df-57cb-8a57-36a271d038be-common/-/monaco-vscode-897bebad-39df-57cb-8a57-36a271d038be-common-21.6.0.tgz", + "integrity": "sha512-yvnqdBKjWcnp4K0rWwIXCecJHGW1JgnkY90xNIDQaQQUT1uFRV/hG99ViS1MFY+mTg8XoDXwwjNZhArk3r4iYQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-89a82baf-8ded-5b2f-b8af-e5fbd72dc5ad-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-89a82baf-8ded-5b2f-b8af-e5fbd72dc5ad-common/-/monaco-vscode-89a82baf-8ded-5b2f-b8af-e5fbd72dc5ad-common-20.2.1.tgz", - "integrity": "sha512-rep9v6GgRfM+bwD4xrGenmTrX/luSsOX4xdOK9JQpa7YSdk1lT3M3+/PzIfcLThScikgMPh+qiyg8FaDGcuodA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-89a82baf-8ded-5b2f-b8af-e5fbd72dc5ad-common/-/monaco-vscode-89a82baf-8ded-5b2f-b8af-e5fbd72dc5ad-common-21.6.0.tgz", + "integrity": "sha512-NpnGhCRBB1BzRM5nTutDvrPF/uIGXijD29l9gBCmAlOZHG2Lr22LYs1cbPVncdgtfMOSmLdL8oWMQHt5aC77/w==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-256d5b78-0649-50e9-8354-2807f95f68f4-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-256d5b78-0649-50e9-8354-2807f95f68f4-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-8ccb7637-50ea-5359-97bf-00015d7fe567-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-8ccb7637-50ea-5359-97bf-00015d7fe567-common/-/monaco-vscode-8ccb7637-50ea-5359-97bf-00015d7fe567-common-20.2.1.tgz", - "integrity": "sha512-5IhDRyiRLPAysf2VD4KYrcGGeTkejwNRKoiXDWigd9PXdLcyJwv/ntvkY5KvcXf8CKUX6oBvdK3QYD0VegJmng==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-8ccb7637-50ea-5359-97bf-00015d7fe567-common/-/monaco-vscode-8ccb7637-50ea-5359-97bf-00015d7fe567-common-21.6.0.tgz", + "integrity": "sha512-kvMQJUpTu11zocIzuJq9utPWt17kbMgJIqiGpXBOJ6yZz9Whl3raElmWbw3MhbV/fu+L9GFauoQUzNTtXEJk+A==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-6db1b967-5327-5c5c-8c17-bd92774c0fb2-common": "20.2.1", - "@codingame/monaco-vscode-897bebad-39df-57cb-8a57-36a271d038be-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-6db1b967-5327-5c5c-8c17-bd92774c0fb2-common": "21.6.0", + "@codingame/monaco-vscode-897bebad-39df-57cb-8a57-36a271d038be-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-96e83782-7f38-572e-8787-02e981f1c54f-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-96e83782-7f38-572e-8787-02e981f1c54f-common/-/monaco-vscode-96e83782-7f38-572e-8787-02e981f1c54f-common-20.2.1.tgz", - "integrity": "sha512-m/oReuik0SnYS8tofyH/B9Ne2mufr8wWPNfXNZeODmKgH/4A94IGc45TX+5yxcnENYYlaGb4SRzkTXe/CqzLBg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-96e83782-7f38-572e-8787-02e981f1c54f-common/-/monaco-vscode-96e83782-7f38-572e-8787-02e981f1c54f-common-21.6.0.tgz", + "integrity": "sha512-muL9TTs28G9yQ/CahSm3g3wQqvJV2UlOd/GG1r5x+JfuVm8TKCT1oL3GavlKlc4SeS1P7zOEFqmLqCYYXUufaQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-ebba7d85-8a22-5735-adf4-8299cd976dce-common": "20.2.1", - "@codingame/monaco-vscode-f22e7e55-aee8-5b52-a6bc-950efd9f5890-common": "20.2.1" + "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-ebba7d85-8a22-5735-adf4-8299cd976dce-common": "21.6.0", + "@codingame/monaco-vscode-f22e7e55-aee8-5b52-a6bc-950efd9f5890-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-9a1a5840-af83-5d07-a156-ba32a36c5c4b-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-9a1a5840-af83-5d07-a156-ba32a36c5c4b-common/-/monaco-vscode-9a1a5840-af83-5d07-a156-ba32a36c5c4b-common-20.2.1.tgz", - "integrity": "sha512-KbwZGcSZrc85o/bQx6K+K5+tSmldEykFdwUrsELknfyrXzD7XDUHP4OAPmCB2tyMYx+2CtoDg1Ap8OpqlTM1bg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-9a1a5840-af83-5d07-a156-ba32a36c5c4b-common/-/monaco-vscode-9a1a5840-af83-5d07-a156-ba32a36c5c4b-common-21.6.0.tgz", + "integrity": "sha512-th0S/u7aPgsYcTe6jHECWTw6yIBB2KUvTBEiCY+j/gC1J35v14E5SWXJqRbII9fjZJ10rbcwDS/ZiQNWYK/1LA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" + } + }, + "node_modules/@codingame/monaco-vscode-9a5ab9e7-d838-5831-9eb4-e79ea3764dcb-common": { + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-9a5ab9e7-d838-5831-9eb4-e79ea3764dcb-common/-/monaco-vscode-9a5ab9e7-d838-5831-9eb4-e79ea3764dcb-common-21.6.0.tgz", + "integrity": "sha512-+07rPCeg61ECDig7YCS5+Cc3WL4idNqfaaEvOD41n0+EzdY/vk2esVvYiwrwwUBnzmf8fKAAM5rVR7+d5zrxEg==", + "license": "MIT", + "dependencies": { + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-9a934394-0cf8-512d-939b-77e71f69cebb-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-9a934394-0cf8-512d-939b-77e71f69cebb-common/-/monaco-vscode-9a934394-0cf8-512d-939b-77e71f69cebb-common-20.2.1.tgz", - "integrity": "sha512-rVgILziuffOxYcOu0ILcW8VAEukpm8qEHb2oTbzbM757jyAJMuJDl1Zd13ByJfdTnqE2GSjTvfCrEsejqmIx2Q==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-9a934394-0cf8-512d-939b-77e71f69cebb-common/-/monaco-vscode-9a934394-0cf8-512d-939b-77e71f69cebb-common-21.6.0.tgz", + "integrity": "sha512-noM4NckKtHHDnHo2LSoQyWdLwa+5pzWMLefFavADt8BuOH2ADt38MjiZsaHLWpbSgUpytsx5fPYz9U6inplSEA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-9c84f943-bcb5-5bcf-92a6-91f66a732f26-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-9c84f943-bcb5-5bcf-92a6-91f66a732f26-common/-/monaco-vscode-9c84f943-bcb5-5bcf-92a6-91f66a732f26-common-20.2.1.tgz", - "integrity": "sha512-UWU1oBgAjNP04/spCB5L5EMXlXHpIMyyRFA8qHIoUgYR8nqo3E6bpFPHjt35pCP/nzPvm0QJSbbEA7V9+VuLmQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-9c84f943-bcb5-5bcf-92a6-91f66a732f26-common/-/monaco-vscode-9c84f943-bcb5-5bcf-92a6-91f66a732f26-common-21.6.0.tgz", + "integrity": "sha512-vjDOmNMckbtXBM7D2ImkJ5gjKw6+hS3+5JhdDmVUW5MnrfCTR5qPwwcfe8APoDCnzN+Bx7/HvN7z9Oax3qm+jw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-9d0168a3-519b-57f3-9bcc-89efc41f951a-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-9d0168a3-519b-57f3-9bcc-89efc41f951a-common/-/monaco-vscode-9d0168a3-519b-57f3-9bcc-89efc41f951a-common-20.2.1.tgz", - "integrity": "sha512-mvzerbSVR0s4vW2u0l/Slub+5KkAAJfDjWUr9r83E9TdvfrEpkUahHMNtqyR0tZ7EAb9Bh367vmLeoCi2AWiWA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-9d0168a3-519b-57f3-9bcc-89efc41f951a-common/-/monaco-vscode-9d0168a3-519b-57f3-9bcc-89efc41f951a-common-21.6.0.tgz", + "integrity": "sha512-3kRxkDe1q5pCOJnVaKXaUUjxj2vql37JyjsVQ7zEzMCYXNeUTR/ACmxcLr4RkydYe4qnrRHjLUAyavRbAMeeJg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-9a934394-0cf8-512d-939b-77e71f69cebb-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-9a934394-0cf8-512d-939b-77e71f69cebb-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-9ee79c1a-3f03-568b-8eac-b02513a98b68-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-9ee79c1a-3f03-568b-8eac-b02513a98b68-common/-/monaco-vscode-9ee79c1a-3f03-568b-8eac-b02513a98b68-common-20.2.1.tgz", - "integrity": "sha512-yMGrBy2R4M3pc69BXrGnHc4U6hdeCZRYFz08IlwOqmRIKEvETGEYARLMjNIKF7OSGtN72w/ogYdDw904Sr+Bwg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-9ee79c1a-3f03-568b-8eac-b02513a98b68-common/-/monaco-vscode-9ee79c1a-3f03-568b-8eac-b02513a98b68-common-21.6.0.tgz", + "integrity": "sha512-QsrNwsfenMiviQyR0F9TWQp/mo2pWGFLsALApsWENGdIYbLN811Fch9CE4aZI5GCtwP6A8pHEmJX+x3sBpAmRg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-262ed59d-4f76-57cd-9e9f-1877f26ae049-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-e39a1c8f-7892-5d9b-9987-7b10b79e1a0a-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common/-/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common-20.2.1.tgz", - "integrity": "sha512-07ZlQCLQYnJmdDz4j0aVertHJ6/ddEryLXn6u5LTG/GJNEQzFUUwr3O16i0TLqsCGYaGGZ4R1cZq5EbfF/Og2Q==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common/-/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common-21.6.0.tgz", + "integrity": "sha512-PtSOMlbJArTzHYNPL87YVwNdzQYU2CDFubGRdgl7JnEPVndWtCewPCRZ2X6NfwJUQmffJLrP71BJhXLktvVytw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-501b06ab-3f58-516b-8a1a-c29d375d3da4-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-a8d3bd74-e63e-5327-96e8-4f931661e329-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-bc6d9a89-1625-5010-b57e-ff44151144fe-common": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1", - "@codingame/monaco-vscode-ebba7d85-8a22-5735-adf4-8299cd976dce-common": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-4fad3647-b95d-5c19-bab1-bb9de627a5ec-common": "21.6.0", + "@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common": "21.6.0", + "@codingame/monaco-vscode-71c8dbff-4c98-552f-aef0-e72b00fdcfc0-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-bc6d9a89-1625-5010-b57e-ff44151144fe-common": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0", + "@codingame/monaco-vscode-ebba7d85-8a22-5735-adf4-8299cd976dce-common": "21.6.0" } }, - "node_modules/@codingame/monaco-vscode-a17e9d37-b6c1-5556-8402-5db73960fae3-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-a17e9d37-b6c1-5556-8402-5db73960fae3-common/-/monaco-vscode-a17e9d37-b6c1-5556-8402-5db73960fae3-common-20.2.1.tgz", - "integrity": "sha512-rpurbDFIG1YSOZfciXFquEmOYDGXvfaimVBQpjY6WcF9V6x24KNJHnx3Wa06gaXJoaQ7M4ucRr1kN6ynJKtcNQ==", + "node_modules/@codingame/monaco-vscode-a175bd1a-4858-5944-9ae5-fb73305dcb13-common": { + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-a175bd1a-4858-5944-9ae5-fb73305dcb13-common/-/monaco-vscode-a175bd1a-4858-5944-9ae5-fb73305dcb13-common-21.6.0.tgz", + "integrity": "sha512-B4YG+FTGuwGXzbyiNuhOY2aJOVGwD0sQWPfrIq+pC5IvYJJUxMkCl4Aj/FydtJgcYKqY4k5F9PFcMu2xUOHWSQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-a3eaa464-944c-5b8f-8886-213068ba4897-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-a3eaa464-944c-5b8f-8886-213068ba4897-common/-/monaco-vscode-a3eaa464-944c-5b8f-8886-213068ba4897-common-20.2.1.tgz", - "integrity": "sha512-XurbqNDgfF7Jv5YV9nc1+zQ1C9YTMRgKwN3OojumBpLFdo0VwJ51WRRrLw6OGp5pVEcOQJaL74OUqZlX5ChI5g==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-a3eaa464-944c-5b8f-8886-213068ba4897-common/-/monaco-vscode-a3eaa464-944c-5b8f-8886-213068ba4897-common-21.6.0.tgz", + "integrity": "sha512-0i1ndH8D3dTGAalp/w0om6eRg6VXwbcKdaN+KqQ8zOcxqr354lBO5HFqGYeVBDxJdcIwn0Dp5Qdt7Wfp4VHZNw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-a654b07e-8806-5425-b124-18f03ba8e11a-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-a654b07e-8806-5425-b124-18f03ba8e11a-common/-/monaco-vscode-a654b07e-8806-5425-b124-18f03ba8e11a-common-20.2.1.tgz", - "integrity": "sha512-pENxDpxf2HM93oG7fjI13M/4zxiDZuKnXIzGgfBPchHEQww0+ZBV9fzOHYrQySkntpowD9uoNeikhTKEtYkiIw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-a654b07e-8806-5425-b124-18f03ba8e11a-common/-/monaco-vscode-a654b07e-8806-5425-b124-18f03ba8e11a-common-21.6.0.tgz", + "integrity": "sha512-Zk/WWJ4GReQ+QLVXb/0sQy4zYqfq55JlItyGrrgMczx5M6lAjxhSO+ZDzv7h0/SHWCRdvx2gPkoHlp0pgaUvyA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" - } - }, - "node_modules/@codingame/monaco-vscode-a8d3bd74-e63e-5327-96e8-4f931661e329-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-a8d3bd74-e63e-5327-96e8-4f931661e329-common/-/monaco-vscode-a8d3bd74-e63e-5327-96e8-4f931661e329-common-20.2.1.tgz", - "integrity": "sha512-JQLcPhCfxc4RMsSuIPKzYCXAKVGYNd262kqfjcVLwPznVVcCPbBe4fWu9khKnVEPE9bWDPO9o4iMOuUDLXu8/A==", - "license": "MIT", - "dependencies": { - "@codingame/monaco-vscode-501b06ab-3f58-516b-8a1a-c29d375d3da4-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-a9da9abe-278d-5ce6-9418-99c7c07c5c37-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-a9da9abe-278d-5ce6-9418-99c7c07c5c37-common/-/monaco-vscode-a9da9abe-278d-5ce6-9418-99c7c07c5c37-common-20.2.1.tgz", - "integrity": "sha512-hqFZdyhyaRnetCP6km6f2QD4ZewOe2g6zNC4yX2fvmN1WkvkDiab1HLHEbl0iEcVWH4XB2RXTYAjBK/xkMfEXA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-a9da9abe-278d-5ce6-9418-99c7c07c5c37-common/-/monaco-vscode-a9da9abe-278d-5ce6-9418-99c7c07c5c37-common-21.6.0.tgz", + "integrity": "sha512-bFhByCQ5in7B/pdBb9xhTYdT6lbZrBrRoITQC7/2g3ZMStHVeifTTmTKDoE83xwpe+tUHzhW7DLXbGJDXo5+UA==", "license": "MIT" }, "node_modules/@codingame/monaco-vscode-abed5a84-8a82-5f84-9412-88a736235bae-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-abed5a84-8a82-5f84-9412-88a736235bae-common/-/monaco-vscode-abed5a84-8a82-5f84-9412-88a736235bae-common-20.2.1.tgz", - "integrity": "sha512-wi6SYca8DUN6D2/bjwCc6FxSyr9mB/o8z8JQDu2cd7HYjMg7SwfqLypiaxEOaYhJa84R/Om9s0dbcjBIxndAcQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-abed5a84-8a82-5f84-9412-88a736235bae-common/-/monaco-vscode-abed5a84-8a82-5f84-9412-88a736235bae-common-21.6.0.tgz", + "integrity": "sha512-zlO6GjQWM67XpWixPkUySjhMr2MS+w/qQ2Wna4BBtCvJ2eHkmCp4ZDIalrVfsWr1Lk6TW3cs8eOQnBPhpCKiuA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-ac93482b-2178-52df-a200-ba0d1a4963fb-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-ac93482b-2178-52df-a200-ba0d1a4963fb-common/-/monaco-vscode-ac93482b-2178-52df-a200-ba0d1a4963fb-common-20.2.1.tgz", - "integrity": "sha512-rtg073ZjqdDZ8SIy502gMQRxX0S/ZxrqFZX/rWO1lDXagirWNbIhn0vPfwCM8ydCwPf4TEyTaOswflETW1Mu6A==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-ac93482b-2178-52df-a200-ba0d1a4963fb-common/-/monaco-vscode-ac93482b-2178-52df-a200-ba0d1a4963fb-common-21.6.0.tgz", + "integrity": "sha512-TWrhJw1RE9+19cmjv5xlFrmtPEVUS/e6T/rTEyOWFI+5l3RCKXcVI+mo1Y+xGbeT1dyCgzncWnTwGnmH+kmr3Q==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-615ce609-8555-545a-a549-47bd9f80e9f8-common": "20.2.1", - "@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common": "20.2.1", - "@codingame/monaco-vscode-72a1b7d3-3f58-5545-9b7e-f579bd003081-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-bd0792ac-6043-5ec3-a41a-54ccb922a1f4-common": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1", - "@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common": "20.2.1", - "@codingame/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common": "20.2.1" + "@codingame/monaco-vscode-615ce609-8555-545a-a549-47bd9f80e9f8-common": "21.6.0", + "@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common": "21.6.0", + "@codingame/monaco-vscode-72a1b7d3-3f58-5545-9b7e-f579bd003081-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-bd0792ac-6043-5ec3-a41a-54ccb922a1f4-common": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0", + "@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common": "21.6.0", + "@codingame/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-acd79e2c-c7e3-5594-873a-427e3006b3d8-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-acd79e2c-c7e3-5594-873a-427e3006b3d8-common/-/monaco-vscode-acd79e2c-c7e3-5594-873a-427e3006b3d8-common-20.2.1.tgz", - "integrity": "sha512-43OHV7iOAydBSH3UxqHd6tOawkbCF3krswanY93afJA97T8Zyyakn25JG/6yxcdjtDtQZjxkp6YMyGRzBnWErw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-acd79e2c-c7e3-5594-873a-427e3006b3d8-common/-/monaco-vscode-acd79e2c-c7e3-5594-873a-427e3006b3d8-common-21.6.0.tgz", + "integrity": "sha512-ElVJjLIfL9vkun5YzCotZrkKppPNXU4rrNkFZTKvS4qAtobDBDJKGuguIx1+9KttIPYddKTL3qEVosgOesJ8gw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-api": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-api/-/monaco-vscode-api-20.2.1.tgz", - "integrity": "sha512-V0U7srsbChsugxi4HP/Q9Xrzw1eajB3UfGUN7l+EYkZK72kChtXoNTyds1wiUfNwsQIt4C6MIwyOQqOCrPY5Lw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-api/-/monaco-vscode-api-21.6.0.tgz", + "integrity": "sha512-ahQTgSMLx43qUGz+5BM+OkIyIaILN1v1qELSd/tSrsMp0u1Z9ttLM7TNUwi8Sd2f6PnX+oURnTQKu8e9/fD/rQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-base-service-override": "20.2.1", - "@codingame/monaco-vscode-environment-service-override": "20.2.1", - "@codingame/monaco-vscode-extensions-service-override": "20.2.1", - "@codingame/monaco-vscode-files-service-override": "20.2.1", - "@codingame/monaco-vscode-host-service-override": "20.2.1", - "@codingame/monaco-vscode-layout-service-override": "20.2.1", - "@codingame/monaco-vscode-quickaccess-service-override": "20.2.1", + "@codingame/monaco-vscode-base-service-override": "21.6.0", + "@codingame/monaco-vscode-environment-service-override": "21.6.0", + "@codingame/monaco-vscode-extensions-service-override": "21.6.0", + "@codingame/monaco-vscode-files-service-override": "21.6.0", + "@codingame/monaco-vscode-host-service-override": "21.6.0", + "@codingame/monaco-vscode-layout-service-override": "21.6.0", + "@codingame/monaco-vscode-quickaccess-service-override": "21.6.0", "@vscode/iconv-lite-umd": "0.7.0", - "dompurify": "3.2.6", + "dompurify": "3.2.7", "jschardet": "3.1.4", "marked": "14.0.0" } }, - "node_modules/@codingame/monaco-vscode-b994942c-360d-5b68-8a33-77d4bde6b714-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-b994942c-360d-5b68-8a33-77d4bde6b714-common/-/monaco-vscode-b994942c-360d-5b68-8a33-77d4bde6b714-common-20.2.1.tgz", - "integrity": "sha512-D5q+XtmzMFHwVX9/2kp8woXMhnfGaRwv5CBWxQM4SGRJzfbDE+43K1ngNWazyK8r5+7QUYEjJmVWsDeh0LQTCQ==", + "node_modules/@codingame/monaco-vscode-b6d52a6d-8c8e-51f5-bcd2-1722295e31d9-common": { + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-b6d52a6d-8c8e-51f5-bcd2-1722295e31d9-common/-/monaco-vscode-b6d52a6d-8c8e-51f5-bcd2-1722295e31d9-common-21.6.0.tgz", + "integrity": "sha512-Ah1AMbSoqS8/icVMPj5Y1cPFu23SVIWfjFq5ApIEQ2pJkI5bifcqO1Jx1AQNEVM4HX/fG1AlyXQFo8t6Bt0mBA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1" + "@codingame/monaco-vscode-a175bd1a-4858-5944-9ae5-fb73305dcb13-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" + } + }, + "node_modules/@codingame/monaco-vscode-b994942c-360d-5b68-8a33-77d4bde6b714-common": { + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-b994942c-360d-5b68-8a33-77d4bde6b714-common/-/monaco-vscode-b994942c-360d-5b68-8a33-77d4bde6b714-common-21.6.0.tgz", + "integrity": "sha512-EmczqQYtSuzCH/UHHyvrrUMBbRXvKcwHKNZEHpprROiuZboyTACfMLN9dLbEG5sWCk8Hr9BBvPwPzVqpsYj0aA==", + "license": "MIT", + "dependencies": { + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-base-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-base-service-override/-/monaco-vscode-base-service-override-20.2.1.tgz", - "integrity": "sha512-U+XrQIXmhwrKwO/wntLxk9ZySMviDFd+1XbjCuZFBs++SYYk8p7vul93SyfAs7MVz37Vdp13XULJzYqZObEDDw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-base-service-override/-/monaco-vscode-base-service-override-21.6.0.tgz", + "integrity": "sha512-9pdCH0wghJPV7jfpWEp92vyWoiGAcknvUyDvoSpYBcackpB96w8CfyN+ljhw1GMeU4lu4o2sDQByJI3Uk3XrKw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "20.2.1", - "@codingame/monaco-vscode-23aade48-f094-5c08-9555-97fc9cca96c9-common": "20.2.1", - "@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-9a1a5840-af83-5d07-a156-ba32a36c5c4b-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-d987325e-3e05-53aa-b9ff-6f97476f64db-common": "20.2.1" + "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "21.6.0", + "@codingame/monaco-vscode-23aade48-f094-5c08-9555-97fc9cca96c9-common": "21.6.0", + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common": "21.6.0", + "@codingame/monaco-vscode-9a1a5840-af83-5d07-a156-ba32a36c5c4b-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-d987325e-3e05-53aa-b9ff-6f97476f64db-common": "21.6.0" + } + }, + "node_modules/@codingame/monaco-vscode-bba55be6-41a2-50cd-a3cc-8bafa35bfa89-common": { + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-bba55be6-41a2-50cd-a3cc-8bafa35bfa89-common/-/monaco-vscode-bba55be6-41a2-50cd-a3cc-8bafa35bfa89-common-21.6.0.tgz", + "integrity": "sha512-+i+k9um6w4zJYw5DtV4Q+o0nX1WgOneKstRDt4YB0+TxAHQ+wu7bpnWaLjE2nshTl86k/l8i+vQmtpQvIS+qjA==", + "license": "MIT", + "dependencies": { + "@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-bc6d9a89-1625-5010-b57e-ff44151144fe-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-bc6d9a89-1625-5010-b57e-ff44151144fe-common/-/monaco-vscode-bc6d9a89-1625-5010-b57e-ff44151144fe-common-20.2.1.tgz", - "integrity": "sha512-rPJKfOtFMZ+rX9uSj0ibs78odA3WwdT16QMDBvMx+l9KeTB3axNONSo577CHsx/zaTK98cwAhJD8gh4iyA/iKQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-bc6d9a89-1625-5010-b57e-ff44151144fe-common/-/monaco-vscode-bc6d9a89-1625-5010-b57e-ff44151144fe-common-21.6.0.tgz", + "integrity": "sha512-34b5zU0Y1xdhtFTr+xtBhWsvZO+ZDC/V1S5BspR3GEfs/NmWH8yP0dGcE9cf3o3dCVIvWknrimb+0v3B7Z1JEA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-a17e9d37-b6c1-5556-8402-5db73960fae3-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-bd0792ac-6043-5ec3-a41a-54ccb922a1f4-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-bd0792ac-6043-5ec3-a41a-54ccb922a1f4-common/-/monaco-vscode-bd0792ac-6043-5ec3-a41a-54ccb922a1f4-common-20.2.1.tgz", - "integrity": "sha512-iQlIMzfM0McG9694DOGeChv5VSIBT2i3Ex56xUgBFFipmpn7GQTZKrtkl1UydhpW1IaP4T6rzV5TOvB4IrKegQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-bd0792ac-6043-5ec3-a41a-54ccb922a1f4-common/-/monaco-vscode-bd0792ac-6043-5ec3-a41a-54ccb922a1f4-common-21.6.0.tgz", + "integrity": "sha512-5dq/h54RGU+3iyqcktc6pOFErEH3QiM8eFs5xx12Ij3mP0d/rBodSUBEoyk0cUSBB9MKn0Gkv0cOszUNTRbjPQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-501b06ab-3f58-516b-8a1a-c29d375d3da4-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-615ce609-8555-545a-a549-47bd9f80e9f8-common": "20.2.1", - "@codingame/monaco-vscode-72a1b7d3-3f58-5545-9b7e-f579bd003081-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1", - "@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common": "20.2.1", - "@codingame/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-4fad3647-b95d-5c19-bab1-bb9de627a5ec-common": "21.6.0", + "@codingame/monaco-vscode-615ce609-8555-545a-a549-47bd9f80e9f8-common": "21.6.0", + "@codingame/monaco-vscode-72a1b7d3-3f58-5545-9b7e-f579bd003081-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0", + "@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common": "21.6.0", + "@codingame/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-be143a32-d60a-5489-a1d2-c83ea7eff6bf-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-be143a32-d60a-5489-a1d2-c83ea7eff6bf-common/-/monaco-vscode-be143a32-d60a-5489-a1d2-c83ea7eff6bf-common-20.2.1.tgz", - "integrity": "sha512-Goq9Tt1fVdigF8oVnd8lvtViZ06hadFoon2kAtz3AehVraYzrs3DfZVufdT0hLOaaxvPT+74iYitl/LDDas29w==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-be143a32-d60a-5489-a1d2-c83ea7eff6bf-common/-/monaco-vscode-be143a32-d60a-5489-a1d2-c83ea7eff6bf-common-21.6.0.tgz", + "integrity": "sha512-b1YKi0NRVykdyBrlzPpHQkiUUamHPGaVwhG8krededphKS0C33SSHITaaUT0c+wd4J2trf9Bc336qxruWqw1/A==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-9a934394-0cf8-512d-939b-77e71f69cebb-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-9a934394-0cf8-512d-939b-77e71f69cebb-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-bf94ddb5-e436-506a-9763-5ab86b642508-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-bf94ddb5-e436-506a-9763-5ab86b642508-common/-/monaco-vscode-bf94ddb5-e436-506a-9763-5ab86b642508-common-20.2.1.tgz", - "integrity": "sha512-8Yju8mAt25Z7bygULthr3nal7/kjf2OYBZDy71A5+NNxeOdlkhxPvlawaJ2Xr2mVlKstBBJ15vvcyPStdEHJSw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-bf94ddb5-e436-506a-9763-5ab86b642508-common/-/monaco-vscode-bf94ddb5-e436-506a-9763-5ab86b642508-common-21.6.0.tgz", + "integrity": "sha512-OBtyw5KXu8JidkjL92OxAuU0DX3cycDVa/jdc8Gdgj9REmcO3qg9WXFY4A0M9gyFHezwNRn1QL+Fv1ws9pcc6Q==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-bulk-edit-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-bulk-edit-service-override/-/monaco-vscode-bulk-edit-service-override-20.2.1.tgz", - "integrity": "sha512-2svGQ9o//PWQCLkqDVK4tMN68t2xKUUb1aCRLxidROtntUZOUxeHyhohk4AQtTHemrHMCaL9Z3cSl3rj5Q6daw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-bulk-edit-service-override/-/monaco-vscode-bulk-edit-service-override-21.6.0.tgz", + "integrity": "sha512-YWXFSNn5JcwOXxaX4+hlCOoArgyj4Z1atk+yUhWPFYY+ifP1qf6czXPbGiDUAAGShn5W8oyv8NgEFgPo5CUNMA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-501b06ab-3f58-516b-8a1a-c29d375d3da4-common": "20.2.1", - "@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common": "20.2.1", - "@codingame/monaco-vscode-a8d3bd74-e63e-5327-96e8-4f931661e329-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-4fad3647-b95d-5c19-bab1-bb9de627a5ec-common": "21.6.0", + "@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common": "21.6.0", + "@codingame/monaco-vscode-71c8dbff-4c98-552f-aef0-e72b00fdcfc0-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common/-/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common-20.2.1.tgz", - "integrity": "sha512-/RVKGRcuaRzXPEr+tHATIXu0441ZG99YMZCETDiSfleQ03DuB7itdW5nMj9pKzKGKXqenWgcRKV1pqJCfpMbjw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common/-/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common-21.6.0.tgz", + "integrity": "sha512-4CJ4iHMUCb/lSol0jW4GHGXflEbHWsqsWeeIfb0IV86c8ZawnoeqqYSwCdktb4CB0zrI3008yyp03ZqGbVH8Yg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-ce7c734f-7712-563c-9335-d7acb43306af-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-ce7c734f-7712-563c-9335-d7acb43306af-common/-/monaco-vscode-ce7c734f-7712-563c-9335-d7acb43306af-common-20.2.1.tgz", - "integrity": "sha512-KvhAG6OM3Xi+aTBA10u2bjw/ClHerBbStouoXZwLozKBXJLrawk5JrP9lyc3rDeMd865uGNe2GDdlT4MnXzQsA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-ce7c734f-7712-563c-9335-d7acb43306af-common/-/monaco-vscode-ce7c734f-7712-563c-9335-d7acb43306af-common-21.6.0.tgz", + "integrity": "sha512-24VY3rsu9pCRo0D00vFrAP8gIGoqJ5FdUf/s5xgZ2n9/ijAJsWm23GAXMx0fKHvEDrEo3ow5yApNhcS0K+eSWg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-cea4d01f-6526-5c2f-8b09-b168fead499f-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-cea4d01f-6526-5c2f-8b09-b168fead499f-common/-/monaco-vscode-cea4d01f-6526-5c2f-8b09-b168fead499f-common-20.2.1.tgz", - "integrity": "sha512-X6pMLGw7VP/q8R+HoULf0sOlSDkwOWJg7s9SBetHY02cpfllmtDHKfZlB/DyxwfIwChypZebYvEzwjWSAzk0IQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-cea4d01f-6526-5c2f-8b09-b168fead499f-common/-/monaco-vscode-cea4d01f-6526-5c2f-8b09-b168fead499f-common-21.6.0.tgz", + "integrity": "sha512-FRQ8VHvAPtyc2oS1WL05VnB1VxmHpGto2rJhJqM+9oq4GnQLOtRFLd6tEZg2wUkAXtt93E/CvwQMsWUPbe0x+w==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-configuration-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-configuration-service-override/-/monaco-vscode-configuration-service-override-20.2.1.tgz", - "integrity": "sha512-Cak6szK7coRFQBi7YTbjqeSV2RUvFMaZaLkmZn7TWoSJleNwwUGotAE3Gl3f1dctXtKXXq9lh//h6PXWwviqkw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-configuration-service-override/-/monaco-vscode-configuration-service-override-21.6.0.tgz", + "integrity": "sha512-E/Vd+t1TTKcQhrlgksVJXQJCmArFo+NdcpCJxCR7MKVUBux63QUgWKuKh0fSxpwkBt1gu1O9+SEoSjC10shN2w==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "20.2.1", - "@codingame/monaco-vscode-422642f2-7e3a-5c1c-9e1e-1d3ef1817346-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-ce7c734f-7712-563c-9335-d7acb43306af-common": "20.2.1", - "@codingame/monaco-vscode-d987325e-3e05-53aa-b9ff-6f97476f64db-common": "20.2.1", - "@codingame/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common": "20.2.1", - "@codingame/monaco-vscode-files-service-override": "20.2.1" + "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "21.6.0", + "@codingame/monaco-vscode-422642f2-7e3a-5c1c-9e1e-1d3ef1817346-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-ce7c734f-7712-563c-9335-d7acb43306af-common": "21.6.0", + "@codingame/monaco-vscode-d987325e-3e05-53aa-b9ff-6f97476f64db-common": "21.6.0", + "@codingame/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common": "21.6.0", + "@codingame/monaco-vscode-files-service-override": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-d0569cfb-4706-5ad6-b0b0-5115ad8685db-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-d0569cfb-4706-5ad6-b0b0-5115ad8685db-common/-/monaco-vscode-d0569cfb-4706-5ad6-b0b0-5115ad8685db-common-20.2.1.tgz", - "integrity": "sha512-KiUFtr157EffTrqbBbXIfHz1Zsykb6qDJIaR/ltl3Zfux6cncEE6hxvAlqdPSK/o+/122UnrBYCFBH8MQUOoCQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-d0569cfb-4706-5ad6-b0b0-5115ad8685db-common/-/monaco-vscode-d0569cfb-4706-5ad6-b0b0-5115ad8685db-common-21.6.0.tgz", + "integrity": "sha512-r6i1KuU3ahZLyP4ESsYVI4DsOsysGoZ73U6vcpCMfMoNuYEG0QAnXbXBcKHQamX+DSfVzjk5I1IFPk2n9edCMw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-0af61f78-dfc5-57ba-8d32-66268c8de38d-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-b994942c-360d-5b68-8a33-77d4bde6b714-common": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1" + "@codingame/monaco-vscode-0af61f78-dfc5-57ba-8d32-66268c8de38d-common": "21.6.0", + "@codingame/monaco-vscode-9a5ab9e7-d838-5831-9eb4-e79ea3764dcb-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-b994942c-360d-5b68-8a33-77d4bde6b714-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-d26a96d3-122c-5a3d-a04d-deb5ff0f19c0-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-d26a96d3-122c-5a3d-a04d-deb5ff0f19c0-common/-/monaco-vscode-d26a96d3-122c-5a3d-a04d-deb5ff0f19c0-common-20.2.1.tgz", - "integrity": "sha512-KlMy6MEPF1kVhEVfyUPfROe78cZxVOHlaxH2uO3D9D6kkqqJLRqAv79wBzkV8ET1BiIQ58r4qtOENxY8um4rPQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-d26a96d3-122c-5a3d-a04d-deb5ff0f19c0-common/-/monaco-vscode-d26a96d3-122c-5a3d-a04d-deb5ff0f19c0-common-21.6.0.tgz", + "integrity": "sha512-CgujHgZZ/e3I+2kcaFHKn2pbS4PDhLK2sWXiaArMsi/+0xlfc9tj2awW7Xz9srvJJHQXvYT+//d0Tm5Pcfygpg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-2a94c04a-b85b-5669-b06b-89c1bfa11cb9-common": "20.2.1", - "@codingame/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-2a94c04a-b85b-5669-b06b-89c1bfa11cb9-common": "21.6.0", + "@codingame/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-d481a59e-259c-524e-bee1-76483d75d3a1-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-d481a59e-259c-524e-bee1-76483d75d3a1-common/-/monaco-vscode-d481a59e-259c-524e-bee1-76483d75d3a1-common-20.2.1.tgz", - "integrity": "sha512-4w0UlPtFsDu/R6ouaw2K1sP2izPLY8LyF4ESzYKXHdUjOiIGdg8jwwz8EdDG5/FglO+1t5SLNxbaCTGNNuMjAA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-d481a59e-259c-524e-bee1-76483d75d3a1-common/-/monaco-vscode-d481a59e-259c-524e-bee1-76483d75d3a1-common-21.6.0.tgz", + "integrity": "sha512-b/fV8556/UFVYMOgoSknmi3PnA2Yyu3En6aEHvjR92dpxBJTrlRp6qGKGVra34Q8P0B4pXh8bWECOwjM2F+kHw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-523730aa-81e6-55d7-9916-87ad537fe087-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-72a1b7d3-3f58-5545-9b7e-f579bd003081-common": "20.2.1", - "@codingame/monaco-vscode-ac93482b-2178-52df-a200-ba0d1a4963fb-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1", - "@codingame/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common": "20.2.1" + "@codingame/monaco-vscode-523730aa-81e6-55d7-9916-87ad537fe087-common": "21.6.0", + "@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common": "21.6.0", + "@codingame/monaco-vscode-72a1b7d3-3f58-5545-9b7e-f579bd003081-common": "21.6.0", + "@codingame/monaco-vscode-9a5ab9e7-d838-5831-9eb4-e79ea3764dcb-common": "21.6.0", + "@codingame/monaco-vscode-ac93482b-2178-52df-a200-ba0d1a4963fb-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0", + "@codingame/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-d609a7d3-bf87-551a-884f-550a8b327ec5-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-d609a7d3-bf87-551a-884f-550a8b327ec5-common/-/monaco-vscode-d609a7d3-bf87-551a-884f-550a8b327ec5-common-20.2.1.tgz", - "integrity": "sha512-6uG+28Zj39LD2HrfuMYwIz42imewq0xgKPreI0BoyDqaRNzWkCQlRbP9JDUHK9ZFnfPFImfxrmQJkFx82WWCsA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-d609a7d3-bf87-551a-884f-550a8b327ec5-common/-/monaco-vscode-d609a7d3-bf87-551a-884f-550a8b327ec5-common-21.6.0.tgz", + "integrity": "sha512-XeBEPtN0cqwbTa+r+5uP7EeJq6grMWftRHO2x6h0h+H4DtZyDy6mvzjpdt04TUSh4hOjd7NiOqHJxFpPhKm/wA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" - } - }, - "node_modules/@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common/-/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common-20.2.1.tgz", - "integrity": "sha512-J1GtOd6fiBVV2NWgWtQUgXd2iWE07/23lb8p5cGDChbmvnYReKNLNEmrrnXGRiptN1PokiJXzmanSc9C1dX/Uw==", - "license": "MIT", - "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-d987325e-3e05-53aa-b9ff-6f97476f64db-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-d987325e-3e05-53aa-b9ff-6f97476f64db-common/-/monaco-vscode-d987325e-3e05-53aa-b9ff-6f97476f64db-common-20.2.1.tgz", - "integrity": "sha512-X10AYZvaKJtPC5SpvW3IIu92kZSn4flvpTiD2n8iyPgs8HF98TIfhyYkreR17gfKVbh2tOS0KAMGD3fORV+srA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-d987325e-3e05-53aa-b9ff-6f97476f64db-common/-/monaco-vscode-d987325e-3e05-53aa-b9ff-6f97476f64db-common-21.6.0.tgz", + "integrity": "sha512-QIq/1Cwbzx0i/FG8nkvM5uZKOyV/MCeQrpqRY+EKZBEIOCFULM/NrK/Wvsr7SWl0RLIOgKfkdGdnqRl0Ebm2kA==", "license": "MIT" }, "node_modules/@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common/-/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common-20.2.1.tgz", - "integrity": "sha512-dDTxW7RAUyzksMC7DHrF7Z5z80SdV6rx/aADjmy12OVHTqf4Dltxy73WF0NxIhjSGHVAOqheBQ6kwgl4xqqc0g==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common/-/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common-21.6.0.tgz", + "integrity": "sha512-KPggwPWzoNJ2E2sBnmIqvoeiJIrNAr2gJj75D+wpGNO/nN6IgONxq4Mk78s8HcVACN2WVmE0v9PLFT9E36zlbg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-a17e9d37-b6c1-5556-8402-5db73960fae3-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-bc6d9a89-1625-5010-b57e-ff44151144fe-common": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-bc6d9a89-1625-5010-b57e-ff44151144fe-common": "21.6.0" + } + }, + "node_modules/@codingame/monaco-vscode-e39a1c8f-7892-5d9b-9987-7b10b79e1a0a-common": { + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-e39a1c8f-7892-5d9b-9987-7b10b79e1a0a-common/-/monaco-vscode-e39a1c8f-7892-5d9b-9987-7b10b79e1a0a-common-21.6.0.tgz", + "integrity": "sha512-mdB6ECEtlAVCQv3Uxii0iTOn6CTTxBNnqtEbzHpC3TNmZM1vjaYBgdrVt6gxLpEA6zEEQOl+BlzAw+A7iPq7mw==", + "license": "MIT", + "dependencies": { + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-e59ecb8c-db32-5324-8fe4-cf9921fd92b8-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-e59ecb8c-db32-5324-8fe4-cf9921fd92b8-common/-/monaco-vscode-e59ecb8c-db32-5324-8fe4-cf9921fd92b8-common-20.2.1.tgz", - "integrity": "sha512-6K8xG7abLgJ8aG56YVUp4YAoEBx7ped857dUoNWyRLQg5S5jvsqik3UmeG6eQSXELFFZlWVdRnsW5+XPXGrf7Q==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-e59ecb8c-db32-5324-8fe4-cf9921fd92b8-common/-/monaco-vscode-e59ecb8c-db32-5324-8fe4-cf9921fd92b8-common-21.6.0.tgz", + "integrity": "sha512-Cy1AFPrHzhnN/vox2daQiEzO3mKsUrhnGbaxUafIpIgBukDYewE/odMkEc8jpEeXELACAYoQPw9187MG+1EXFA==", "license": "MIT" }, "node_modules/@codingame/monaco-vscode-e72c94ca-257a-5b75-8b68-5a5fa3c18255-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-e72c94ca-257a-5b75-8b68-5a5fa3c18255-common/-/monaco-vscode-e72c94ca-257a-5b75-8b68-5a5fa3c18255-common-20.2.1.tgz", - "integrity": "sha512-nyW5khTs4AgNxhLuB/wCOKQLvJ0zUI1oJPTVnXTZAghHPj4bKI+Oj5QKg+8extxROuNBAkyDSnWkdsnFcSpRlg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-e72c94ca-257a-5b75-8b68-5a5fa3c18255-common/-/monaco-vscode-e72c94ca-257a-5b75-8b68-5a5fa3c18255-common-21.6.0.tgz", + "integrity": "sha512-gSfBqKAOA/ixweNx33SEhZfhWY+bhAweo3tIFbGuD/70FviTLVtiPxxUcr7RqjwQIYRiB1tKnKtCMb/JZRbnYA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-d609a7d3-bf87-551a-884f-550a8b327ec5-common": "20.2.1" + "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-d609a7d3-bf87-551a-884f-550a8b327ec5-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-eb7d5efd-2e60-59f8-9ba4-9a8ae8cb2957-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-eb7d5efd-2e60-59f8-9ba4-9a8ae8cb2957-common/-/monaco-vscode-eb7d5efd-2e60-59f8-9ba4-9a8ae8cb2957-common-20.2.1.tgz", - "integrity": "sha512-QO6e8HCjoJzT7HQyP1ysqHW+5uGQz3gTZA9F9cqCSzbSdig7SEPZ1CFeZaCcxifTSVe/koel0AVf4c4qDl1Zaw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-eb7d5efd-2e60-59f8-9ba4-9a8ae8cb2957-common/-/monaco-vscode-eb7d5efd-2e60-59f8-9ba4-9a8ae8cb2957-common-21.6.0.tgz", + "integrity": "sha512-uaekjdYuEv+iazR9GgTDii91fSpn0QvExrqOX4GODLXZ7ghaAs/Yl2mEy9ZfOIePvzLHYUH5jdrcLqffMDBVpA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-262ed59d-4f76-57cd-9e9f-1877f26ae049-common": "20.2.1", - "@codingame/monaco-vscode-501b06ab-3f58-516b-8a1a-c29d375d3da4-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-9ee79c1a-3f03-568b-8eac-b02513a98b68-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-4fad3647-b95d-5c19-bab1-bb9de627a5ec-common": "21.6.0", + "@codingame/monaco-vscode-9ee79c1a-3f03-568b-8eac-b02513a98b68-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-e39a1c8f-7892-5d9b-9987-7b10b79e1a0a-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-eba0b9b3-174c-5dae-9867-a37810ca1808-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-eba0b9b3-174c-5dae-9867-a37810ca1808-common/-/monaco-vscode-eba0b9b3-174c-5dae-9867-a37810ca1808-common-20.2.1.tgz", - "integrity": "sha512-fjhb02OSNCCEvzumxpnH7CF+8hV3/8iEFjm1qOTodczWjvdWa+BhaiUEBt4NueQ8IR/8UtIW9Iq1a0TwOJF2xw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-eba0b9b3-174c-5dae-9867-a37810ca1808-common/-/monaco-vscode-eba0b9b3-174c-5dae-9867-a37810ca1808-common-21.6.0.tgz", + "integrity": "sha512-uTcWNKM8Kc8CQ7oIbrXo1zADBpa9Qbrgyt+hL6IjBpgWDfJK7wWncYWVn3sc1dtPlLBbdysWDnLj7Y+YKIOyQQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-ebba7d85-8a22-5735-adf4-8299cd976dce-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-ebba7d85-8a22-5735-adf4-8299cd976dce-common/-/monaco-vscode-ebba7d85-8a22-5735-adf4-8299cd976dce-common-20.2.1.tgz", - "integrity": "sha512-IaEfEIpOatIxm2iJAopML1ocDkAx/kL+GmN8nSEih1q+1JyLjU8qY3hCZratK0rApjKTWnJCVkZ5hvapLsWpJw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-ebba7d85-8a22-5735-adf4-8299cd976dce-common/-/monaco-vscode-ebba7d85-8a22-5735-adf4-8299cd976dce-common-21.6.0.tgz", + "integrity": "sha512-g1LfbRvkxQzSrW7MrGlDAYo6YJ8/XFX4fZ8HZvj3HTgr9jy60l6GMrkhhTmgJjC/fPqqTqdmSSQ1i/JovSo2Hg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-eda30bac-0984-5b42-9362-c68996b85232-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-eda30bac-0984-5b42-9362-c68996b85232-common/-/monaco-vscode-eda30bac-0984-5b42-9362-c68996b85232-common-20.2.1.tgz", - "integrity": "sha512-HRs+VzEls/SYZeNCPXVkNfPH48ODuYLjSPDNGhW0pwaDnX28nmmchXIH6bc6Og4GFtMSn5eRw1pgBNCkAbljNw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-eda30bac-0984-5b42-9362-c68996b85232-common/-/monaco-vscode-eda30bac-0984-5b42-9362-c68996b85232-common-21.6.0.tgz", + "integrity": "sha512-mlq/4ooKh4j7e/YvVuhwLuDZQ71mQCwNEOGVaaXxYJ4C/CRhBR5NN5O+6ADJ7W/Z1bRxqu/ujUVZuA/+2vYKtQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-622c0cca-d5fa-59b6-b730-0715afcf93ee-common": "20.2.1", - "@codingame/monaco-vscode-a17e9d37-b6c1-5556-8402-5db73960fae3-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common": "21.6.0", + "@codingame/monaco-vscode-622c0cca-d5fa-59b6-b730-0715afcf93ee-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-editor-api": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-editor-api/-/monaco-vscode-editor-api-20.2.1.tgz", - "integrity": "sha512-f+e6Lchp/aW2J5lEqkULP8NF4PGRayVG2/X90HN7ydlWAIr/WDZfVDmP2XhmnTZXfAhCoYQWWljtHMQWY7aSyw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-editor-api/-/monaco-vscode-editor-api-21.6.0.tgz", + "integrity": "sha512-YTxKRHe9d4TvyEzWIqLpJXLyZyO4xFlLgrkgHoWBpomm6gIuwaRJJRpapBZf24oG8AhkniZSPg2iv/84M+ho6g==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-5452e2b7-9081-5f95-839b-4ab3544ce28f-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-5452e2b7-9081-5f95-839b-4ab3544ce28f-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-editor-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-editor-service-override/-/monaco-vscode-editor-service-override-20.2.1.tgz", - "integrity": "sha512-Oetg/Ammu5aaBoFkGYTwNN+iTTsRO45eyY66hjjFFGGaLrcKKpFlgpGjZ7uIXcp8P8QCyPBAYHdEeJ2jFKceGg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-editor-service-override/-/monaco-vscode-editor-service-override-21.6.0.tgz", + "integrity": "sha512-SYC1TrH+OkDUwOjlS4gLbj3JVPWx1QsK17gKbeZaVXowzpJoVL73u9rY0GVkL5B9eS5yvhB2N+wKsa0lrO0Hlw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-72a1b7d3-3f58-5545-9b7e-f579bd003081-common": "20.2.1", - "@codingame/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common": "20.2.1", - "@codingame/monaco-vscode-ac93482b-2178-52df-a200-ba0d1a4963fb-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-72a1b7d3-3f58-5545-9b7e-f579bd003081-common": "21.6.0", + "@codingame/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common": "21.6.0", + "@codingame/monaco-vscode-ac93482b-2178-52df-a200-ba0d1a4963fb-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-environment-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-environment-service-override/-/monaco-vscode-environment-service-override-20.2.1.tgz", - "integrity": "sha512-+jJ2UfaiBJF949UYBg8+X9xf/sQGDjXRWQOlbV/084iX8ao0zed5GmCcxkCNQEncWGhoc/79z8F0+BlAJhqsbw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-environment-service-override/-/monaco-vscode-environment-service-override-21.6.0.tgz", + "integrity": "sha512-cSK0CAOH6f9dBYWHilhcYF52IcgEUliuZiFFNwSyo5mjaFXqAXzeU88rnSkkuewgIO6ZUiVZ+ApA00I7X+Kq7Q==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-abed5a84-8a82-5f84-9412-88a736235bae-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-abed5a84-8a82-5f84-9412-88a736235bae-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-extension-api": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-extension-api/-/monaco-vscode-extension-api-20.2.1.tgz", - "integrity": "sha512-K2VFVhQZUpBS+YJRr4DYgvNjelu7dWw81YWg8PwEVj7X8vSd9DqQZbTvudsiJDhlkkY4KWqZQjtITY0fc/X3QQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-extension-api/-/monaco-vscode-extension-api-21.6.0.tgz", + "integrity": "sha512-oslSpuCAZKS88hPx76Cickqd9/z5M1koUkx8UOqnfVIFAemnMHkZWWC4bOa/Q7wFZBy+1qB0/OuhrIBtycj5vA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common": "20.2.1", - "@codingame/monaco-vscode-4a3ac544-9a61-534c-88df-756262793ef7-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-extensions-service-override": "20.2.1" + "@codingame/monaco-vscode-4a3ac544-9a61-534c-88df-756262793ef7-common": "21.6.0", + "@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-extensions-service-override": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-extensions-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-extensions-service-override/-/monaco-vscode-extensions-service-override-20.2.1.tgz", - "integrity": "sha512-+/m/ZrGyBpfj80f2g0U42OSMhSaXCZNULUKyIM+Amm5TqcaonJ6VysGEga48QO3SGHqzdBRSJJRwR5MNhQAbOw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-extensions-service-override/-/monaco-vscode-extensions-service-override-21.6.0.tgz", + "integrity": "sha512-uqPs5NHBypZRlxux0F4emoD03RzXZ19cpWd3FHxFrLERJRB9BUQqJAVpOb0Fwijp6B9KN5DYxRTS4gRbT5jr+w==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-0af61f78-dfc5-57ba-8d32-66268c8de38d-common": "20.2.1", - "@codingame/monaco-vscode-256d5b78-0649-50e9-8354-2807f95f68f4-common": "20.2.1", - "@codingame/monaco-vscode-2a94c04a-b85b-5669-b06b-89c1bfa11cb9-common": "20.2.1", - "@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common": "20.2.1", - "@codingame/monaco-vscode-4a3ac544-9a61-534c-88df-756262793ef7-common": "20.2.1", - "@codingame/monaco-vscode-571c8352-7953-5038-9f09-e03bb6219a0e-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-622c0cca-d5fa-59b6-b730-0715afcf93ee-common": "20.2.1", - "@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common": "20.2.1", - "@codingame/monaco-vscode-6845754f-e617-5ed9-8aaa-6ca3653a9532-common": "20.2.1", - "@codingame/monaco-vscode-6f931a91-88ea-5232-897f-a17ec3929ba5-common": "20.2.1", - "@codingame/monaco-vscode-7f39b6f1-3542-5430-8760-0f404d7a7cee-common": "20.2.1", - "@codingame/monaco-vscode-8ccb7637-50ea-5359-97bf-00015d7fe567-common": "20.2.1", - "@codingame/monaco-vscode-a17e9d37-b6c1-5556-8402-5db73960fae3-common": "20.2.1", - "@codingame/monaco-vscode-a654b07e-8806-5425-b124-18f03ba8e11a-common": "20.2.1", - "@codingame/monaco-vscode-a8d3bd74-e63e-5327-96e8-4f931661e329-common": "20.2.1", - "@codingame/monaco-vscode-a9da9abe-278d-5ce6-9418-99c7c07c5c37-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-b994942c-360d-5b68-8a33-77d4bde6b714-common": "20.2.1", - "@codingame/monaco-vscode-bf94ddb5-e436-506a-9763-5ab86b642508-common": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1", - "@codingame/monaco-vscode-d0569cfb-4706-5ad6-b0b0-5115ad8685db-common": "20.2.1", - "@codingame/monaco-vscode-eb7d5efd-2e60-59f8-9ba4-9a8ae8cb2957-common": "20.2.1", - "@codingame/monaco-vscode-eba0b9b3-174c-5dae-9867-a37810ca1808-common": "20.2.1", - "@codingame/monaco-vscode-f22e7e55-aee8-5b52-a6bc-950efd9f5890-common": "20.2.1", - "@codingame/monaco-vscode-files-service-override": "20.2.1" + "@codingame/monaco-vscode-05a2a821-e4de-5941-b7f9-bbf01c09f229-common": "21.6.0", + "@codingame/monaco-vscode-0af61f78-dfc5-57ba-8d32-66268c8de38d-common": "21.6.0", + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-249dc928-1da3-51c1-82d0-45e0ba9d08a1-common": "21.6.0", + "@codingame/monaco-vscode-256d5b78-0649-50e9-8354-2807f95f68f4-common": "21.6.0", + "@codingame/monaco-vscode-2a94c04a-b85b-5669-b06b-89c1bfa11cb9-common": "21.6.0", + "@codingame/monaco-vscode-4a3ac544-9a61-534c-88df-756262793ef7-common": "21.6.0", + "@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common": "21.6.0", + "@codingame/monaco-vscode-4dda7789-5a25-5e8b-b2de-c2f11b1b96e5-common": "21.6.0", + "@codingame/monaco-vscode-571c8352-7953-5038-9f09-e03bb6219a0e-common": "21.6.0", + "@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common": "21.6.0", + "@codingame/monaco-vscode-622c0cca-d5fa-59b6-b730-0715afcf93ee-common": "21.6.0", + "@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common": "21.6.0", + "@codingame/monaco-vscode-6845754f-e617-5ed9-8aaa-6ca3653a9532-common": "21.6.0", + "@codingame/monaco-vscode-71c8dbff-4c98-552f-aef0-e72b00fdcfc0-common": "21.6.0", + "@codingame/monaco-vscode-7f39b6f1-3542-5430-8760-0f404d7a7cee-common": "21.6.0", + "@codingame/monaco-vscode-8ccb7637-50ea-5359-97bf-00015d7fe567-common": "21.6.0", + "@codingame/monaco-vscode-a654b07e-8806-5425-b124-18f03ba8e11a-common": "21.6.0", + "@codingame/monaco-vscode-a9da9abe-278d-5ce6-9418-99c7c07c5c37-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-b994942c-360d-5b68-8a33-77d4bde6b714-common": "21.6.0", + "@codingame/monaco-vscode-bba55be6-41a2-50cd-a3cc-8bafa35bfa89-common": "21.6.0", + "@codingame/monaco-vscode-bf94ddb5-e436-506a-9763-5ab86b642508-common": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0", + "@codingame/monaco-vscode-d0569cfb-4706-5ad6-b0b0-5115ad8685db-common": "21.6.0", + "@codingame/monaco-vscode-eb7d5efd-2e60-59f8-9ba4-9a8ae8cb2957-common": "21.6.0", + "@codingame/monaco-vscode-eba0b9b3-174c-5dae-9867-a37810ca1808-common": "21.6.0", + "@codingame/monaco-vscode-f22e7e55-aee8-5b52-a6bc-950efd9f5890-common": "21.6.0", + "@codingame/monaco-vscode-files-service-override": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-f1bbc6d3-6129-583c-a2ba-c80b832993d2-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-f1bbc6d3-6129-583c-a2ba-c80b832993d2-common/-/monaco-vscode-f1bbc6d3-6129-583c-a2ba-c80b832993d2-common-20.2.1.tgz", - "integrity": "sha512-1Mv1RgKdwicAEY4aJR/WvI8NOHYt2JcEP4SlyNCLP8op0RJo4brs2Z6p2BqXS+WoHOob1v08lizbCL8RKkFN+g==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-f1bbc6d3-6129-583c-a2ba-c80b832993d2-common/-/monaco-vscode-f1bbc6d3-6129-583c-a2ba-c80b832993d2-common-21.6.0.tgz", + "integrity": "sha512-jiROZSvmiZ/yP5YaeIdexmyUOgOcZ9UMike/ttm8xe6XKiwaO1TAAi8S3499XuR1xff9g2P9KXdPDfsJwi7Xkw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-f22e7e55-aee8-5b52-a6bc-950efd9f5890-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-f22e7e55-aee8-5b52-a6bc-950efd9f5890-common/-/monaco-vscode-f22e7e55-aee8-5b52-a6bc-950efd9f5890-common-20.2.1.tgz", - "integrity": "sha512-HrUgIIQpWkMcGy7fHQFX7y3FOCP0Hetb85lDvZCSf+XiXcz/4YP193bHp3BA+zwRFanD7RWTSrwXFbC3BVPnlQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-f22e7e55-aee8-5b52-a6bc-950efd9f5890-common/-/monaco-vscode-f22e7e55-aee8-5b52-a6bc-950efd9f5890-common-21.6.0.tgz", + "integrity": "sha512-Td9EHwGwf5dx/chrcBAV7ip2lL98Z/S0nyoCv8zkVLwZgs1+ozRk7AYfnYYcGUBhzKbu/P/RPtqnn1Ha4NPtYQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common": "21.6.0", + "@codingame/monaco-vscode-9a5ab9e7-d838-5831-9eb4-e79ea3764dcb-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common/-/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common-20.2.1.tgz", - "integrity": "sha512-R+8qheGjNfJthnO4Qr0zs+x7Ylcw5/g7One5V/izemVoK2hPC4yGSRyF0V1OcAjkqDxJsOvcSU0GrstEVnOI0g==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common/-/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common-21.6.0.tgz", + "integrity": "sha512-3XWFSifzoxUI++i801KWjKXG9lWGKjBsx7jNsa2cl8hERYGvSUgBu2QxXCJhKWda9x1k61z1P6G7t8il9eFlQw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" - } - }, - "node_modules/@codingame/monaco-vscode-f6f55824-df83-5ffc-ac26-50fd4df4fe0e-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-f6f55824-df83-5ffc-ac26-50fd4df4fe0e-common/-/monaco-vscode-f6f55824-df83-5ffc-ac26-50fd4df4fe0e-common-20.2.1.tgz", - "integrity": "sha512-5nTqGRL3UcE6bn/2SXU5fFuCWjyi79OVaOmkrcGwcosfO0oL6Xgw0cU5QSymBi3sb3a7FqhvmA8YWmb1k68NEA==", - "license": "MIT", - "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-ff9fa663-eae3-5274-8573-c2b918871e4b-common": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-ff9fa663-eae3-5274-8573-c2b918871e4b-common/-/monaco-vscode-ff9fa663-eae3-5274-8573-c2b918871e4b-common-20.2.1.tgz", - "integrity": "sha512-FpyXwRDuYxJ55riSDTpxW35tKm5MMZCPymR8bizI3SoZG6FtoKZpTj1ey/NdTMfGT7NC7cyKO2NK8KyvSB95Rw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-ff9fa663-eae3-5274-8573-c2b918871e4b-common/-/monaco-vscode-ff9fa663-eae3-5274-8573-c2b918871e4b-common-21.6.0.tgz", + "integrity": "sha512-Intx8vVL0z4kIwfVMwy23kZiYpXFkBfQ24My1Cn/nKej5nb8lL86tEeJ/F5KnJ72gh1FeehSj5XH8BqX4pxhHg==", "license": "MIT" }, "node_modules/@codingame/monaco-vscode-files-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-files-service-override/-/monaco-vscode-files-service-override-20.2.1.tgz", - "integrity": "sha512-p+3Ycbc5VOwOxH5QkhKvgDavOhGQCk5e7aa4Z0ERtkWst5CF8q0xqQbcke1UBjwLbst2Pt0kRuGXz7FolikzZA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-files-service-override/-/monaco-vscode-files-service-override-21.6.0.tgz", + "integrity": "sha512-h+Ew3dLoJ/Ii3vnkp2f1NSzmD39rB+n1eEhq27RaFSt0iDcnNs6QJYeoSzawi9QyvzCHwyLMM4VwDUNT7FgePA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-0c06bfba-d24d-5c4d-90cd-b40cefb7f811-common": "20.2.1", - "@codingame/monaco-vscode-15626ec7-b165-51e1-8caf-7bcc2ae9b95a-common": "20.2.1", - "@codingame/monaco-vscode-2f06fe84-148e-5e6b-a7ca-c7989c5f128a-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1", - "@codingame/monaco-vscode-cea4d01f-6526-5c2f-8b09-b168fead499f-common": "20.2.1" + "@codingame/monaco-vscode-0c06bfba-d24d-5c4d-90cd-b40cefb7f811-common": "21.6.0", + "@codingame/monaco-vscode-15626ec7-b165-51e1-8caf-7bcc2ae9b95a-common": "21.6.0", + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-2f06fe84-148e-5e6b-a7ca-c7989c5f128a-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0", + "@codingame/monaco-vscode-cea4d01f-6526-5c2f-8b09-b168fead499f-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-host-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-host-service-override/-/monaco-vscode-host-service-override-20.2.1.tgz", - "integrity": "sha512-Q3jSzf39M8dMPNbsmrHaJRDBvIX+9ZzEoieXZf5HnK6UBxULKLXdj0mO1xzZD/4AfxfMQTumHAG8PNkVPJ2NnQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-host-service-override/-/monaco-vscode-host-service-override-21.6.0.tgz", + "integrity": "sha512-DpZGpBP3g4SbEpVNnPykOxMWD/fBSgFTUkEEUZQmoHFAecCfjHl3TBpjcv+DKRRHDnSUC3kz3d8jRKzBw0XXUA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "20.2.1", - "@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "21.6.0", + "@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common": "21.6.0", + "@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-keybindings-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-keybindings-service-override/-/monaco-vscode-keybindings-service-override-20.2.1.tgz", - "integrity": "sha512-+V8GdYynALBXCuJFSkxxAHaEQ9aNve46hVaIl/vffSKwC3ZnKqsnDAdEBCqj0Jlm9oqZZSdAg1iu3SqgFx/OYA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-keybindings-service-override/-/monaco-vscode-keybindings-service-override-21.6.0.tgz", + "integrity": "sha512-6nKsbWE9krjM94IUyCRrpUMHhkwZuFcM1zZOxJ/DCnQ8uXYOWYsw94lNA18QqMS9uSLi6kxCg4DNeyk7vSoUrA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-2a22c7b4-b906-5914-8cd1-3ed912fb738f-common": "20.2.1", - "@codingame/monaco-vscode-40cada32-7e9c-528a-81fc-766e4da54147-common": "20.2.1", - "@codingame/monaco-vscode-a3eaa464-944c-5b8f-8886-213068ba4897-common": "20.2.1", - "@codingame/monaco-vscode-acd79e2c-c7e3-5594-873a-427e3006b3d8-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-d609a7d3-bf87-551a-884f-550a8b327ec5-common": "20.2.1", - "@codingame/monaco-vscode-files-service-override": "20.2.1" + "@codingame/monaco-vscode-2a22c7b4-b906-5914-8cd1-3ed912fb738f-common": "21.6.0", + "@codingame/monaco-vscode-40cada32-7e9c-528a-81fc-766e4da54147-common": "21.6.0", + "@codingame/monaco-vscode-a3eaa464-944c-5b8f-8886-213068ba4897-common": "21.6.0", + "@codingame/monaco-vscode-acd79e2c-c7e3-5594-873a-427e3006b3d8-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-d609a7d3-bf87-551a-884f-550a8b327ec5-common": "21.6.0", + "@codingame/monaco-vscode-files-service-override": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-language-pack-cs": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-cs/-/monaco-vscode-language-pack-cs-20.2.1.tgz", - "integrity": "sha512-t8eith2l7luL9+3ePmxF804E5X3bv2L/LTjg4oKknVnqNjui4GER/7IckWKCX58JT60ZYEuwUBzWgsnnY8gwWQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-cs/-/monaco-vscode-language-pack-cs-21.6.0.tgz", + "integrity": "sha512-+xx2VXCpWZtjMqXZh1aOZYCOVi5ZUGd7pMTXTSx3CU6zbq6a3hhceUZGyI5I3EdRBOAOF6ciDPd3D+8B/B/wiQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-language-pack-de": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-de/-/monaco-vscode-language-pack-de-20.2.1.tgz", - "integrity": "sha512-f7FSLTI/cROB298xbuEVlz4oHxnKcYV8ilDD5wruTsol9S00+rw9KBza4ffkXOAq7PTywtTA0OuPWNi6PxfSmA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-de/-/monaco-vscode-language-pack-de-21.6.0.tgz", + "integrity": "sha512-AEBmmDuBlZQR1yTH8nic7NQkSQqVLjJAYjU7mnC6Z1ZkNsWzUA3zOhAjv6O5kbQhtOtwPm3lc22SFR7ElDid6A==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-language-pack-es": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-es/-/monaco-vscode-language-pack-es-20.2.1.tgz", - "integrity": "sha512-OqtQKZ+8PgFkG1OEf8vL2fGSMBJf9+flcIIpEkcSHLEBcsh0Ki1vJJacIuhgqntUM7n2QucXcEIL+QWhTZjhZA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-es/-/monaco-vscode-language-pack-es-21.6.0.tgz", + "integrity": "sha512-/rizSv3rMdvZVxEbGnQo5m6PeCsh4aLeunc/4Oiqmc9vYsVg3gLr9cwqwQJ0jBgeUS0Xfc1UHuszHYOLdkGFOg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-language-pack-fr": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-fr/-/monaco-vscode-language-pack-fr-20.2.1.tgz", - "integrity": "sha512-/iJFlTb7XL0H2JtxbVo8To67r+U8rQHYJbU6tcOKJ0sNx70RucgsvSXBRvvKkNVcWgRO/qkn3m7Zg275EXG1Qw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-fr/-/monaco-vscode-language-pack-fr-21.6.0.tgz", + "integrity": "sha512-ObfmJXT+YEcQ/H5s38BQctA9rCAYjb6EbxcoyUrYLQo1UrgE94qlgUC3bLOlpmGuqNPyp1XqHNMnIEU+xipbnQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-language-pack-it": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-it/-/monaco-vscode-language-pack-it-20.2.1.tgz", - "integrity": "sha512-ioY7ILE4NH0D6LEHkwQTTipiul8/fRkkAgpZfYpiudxZ0/Wug+rA5KwLt+FyU//KXoRUsaESbeqV2a2bKQq5CA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-it/-/monaco-vscode-language-pack-it-21.6.0.tgz", + "integrity": "sha512-9nFyW7mzWz7iR2dPos1sQdNRU/rgpXuZrGzCc2Fx2XKNpmjP5aTyl2bthrCxANFqOiSyjpphI2op3T4JHxvGBw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-language-pack-ja": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-ja/-/monaco-vscode-language-pack-ja-20.2.1.tgz", - "integrity": "sha512-eZRR3kEpS13EMJZ3CSFi2Nk3tRFAtljp+FcTBdk+CpDy7QysBHfoFaAnHWQXgkBm4relsO++jBGs5lD2jNFggA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-ja/-/monaco-vscode-language-pack-ja-21.6.0.tgz", + "integrity": "sha512-ymm45QHA3ohaFEpLDpoLRVgVpASG2vnxIHnSLSg3U3rOU1rMl/UQvcz6Hp9AoVyxGBqGNLNmaVVQqnu7FF2ZgA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-language-pack-ko": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-ko/-/monaco-vscode-language-pack-ko-20.2.1.tgz", - "integrity": "sha512-silEXtsGwq9QhRihW427RjkrxtUa0grKXFGcSCa5XMZ4VtdRtc1H7zyRVeSSWas/1AMzB1NjIO/Jf7YkYYet8g==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-ko/-/monaco-vscode-language-pack-ko-21.6.0.tgz", + "integrity": "sha512-Qc7oO5z4nUhroA0h+TarN3olz6hRM5kXlW75RoQvzUkEQ52TOMRDYYukNw548f4B+3Rg0hmn0AiAtFGTGz2QHw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-language-pack-pl": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-pl/-/monaco-vscode-language-pack-pl-20.2.1.tgz", - "integrity": "sha512-tDXguYEM2GsfNFUxMf5xvbH6WqV57fw2py4i2FTKvUt6oig4Ijwe/uqIhnC3yokLHhlB+ejpr6wvT0GYJPrKvw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-pl/-/monaco-vscode-language-pack-pl-21.6.0.tgz", + "integrity": "sha512-7FPsViYJ6M7PzO25kNQUyamg7ZqPGlulh3BNH7S7/13KhIt1pOxLmn5r9Dx6rfyERPB9evU1BJ9HJkT5qFJ3aQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-language-pack-pt-br": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-pt-br/-/monaco-vscode-language-pack-pt-br-20.2.1.tgz", - "integrity": "sha512-mBgiUto9JrW1mLtM83u8wgV23uhOsmc8Rn+OcRcwV3WkoOckmLfkTGMWGi63z+wGtjBCQjZnKVJA7Yi97LeQJA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-pt-br/-/monaco-vscode-language-pack-pt-br-21.6.0.tgz", + "integrity": "sha512-xEespwZksuiSzSLarqco/BCnTld6IRXRzLXa9wu1Y1e/2s2quOuxFGltUy4EmlbUfgQuMVsyt0na/CbbxVDdEA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-language-pack-qps-ploc": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-qps-ploc/-/monaco-vscode-language-pack-qps-ploc-20.2.1.tgz", - "integrity": "sha512-VVciG7472zKhy9ELjvRUt/xbigL2wpqt0zSVOAbOgCnX9pyoild4K1AcGHkVtz5i7AeZmpMUF/JeUPb+so5+Tg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-qps-ploc/-/monaco-vscode-language-pack-qps-ploc-21.6.0.tgz", + "integrity": "sha512-cqyaLh7/U5vajyAvZKx0pEaH+psQKEMAorvR2b2iyh7ujr5s6U9YRehziwaZ49jqgeNUn1Rmd5gJD34iyja+iw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-language-pack-ru": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-ru/-/monaco-vscode-language-pack-ru-20.2.1.tgz", - "integrity": "sha512-XvAnq1JzDGShytAtaBuWr92cAAleX+gVTe6mAq1hleeN3bQ2YQBU6Gox+t1WKGoHFPXhuA4QlhUH+5vNTbXDrg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-ru/-/monaco-vscode-language-pack-ru-21.6.0.tgz", + "integrity": "sha512-mXsEDyuFiOwvcF/MM0Yb9vdmUxVoiHZUp6XFsnJbMl7pt5DUNoTfrwNhIVbAjSATn4SdYiJ7sgwXI1Jw5j8EWg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-language-pack-tr": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-tr/-/monaco-vscode-language-pack-tr-20.2.1.tgz", - "integrity": "sha512-ZSYSU1hWt2a60NyAPFdnN4RppQ9TwNJ76O/gX8OJBobOZ7BW98UAjNeGAwtEfHvqnywSgpaMoVZaRhieGvoUSw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-tr/-/monaco-vscode-language-pack-tr-21.6.0.tgz", + "integrity": "sha512-1yABKhZVLjtnP8d2IqSqxmNcJpR+TcYIL+w3UhxqqQEzCy0kK97hE/Ms6m0aMuOrU8tpYHKXp0E1jxBVcvoOrA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-language-pack-zh-hans": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-zh-hans/-/monaco-vscode-language-pack-zh-hans-20.2.1.tgz", - "integrity": "sha512-T4TN6Tq6oEnNnfiBqZEXw54/pSmpkn41DpIqHYago9bEaOCoampW5FEMmyjR5PbQf18jQN92w6cfHg60iCqjEg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-zh-hans/-/monaco-vscode-language-pack-zh-hans-21.6.0.tgz", + "integrity": "sha512-V4AXR+Dq9plOfeHYA4SwFzpb1jHL+DJQrDnUsSUVb6nOaEyblRunNzL4okg/AAiNP6wf3S0ZMZrK5rheFl7e4A==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-language-pack-zh-hant": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-zh-hant/-/monaco-vscode-language-pack-zh-hant-20.2.1.tgz", - "integrity": "sha512-hrwCULz6adfzkvxLjcZELoJ4OWDOlCKhs7BeWc3lYqPsYYmEUrcZcv9gsJ0GTf9zmV9f0aF4V791B1CpzxjcNQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-language-pack-zh-hant/-/monaco-vscode-language-pack-zh-hant-21.6.0.tgz", + "integrity": "sha512-43SUv4m33v2pMetEjt4ABzLdsbGR4XfGpKf/dUmOWTg0ijoPwIq+0fa/CKoC1X7oEfxHp5gQPaFdtx3pZSjQpA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-languages-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-languages-service-override/-/monaco-vscode-languages-service-override-20.2.1.tgz", - "integrity": "sha512-q8MVHEOb4o7p2YMdvW5q3e6U+yL6BVHQJB8JqHgmQ0eTx5gK99PtOCpIst3Tot4CmLiCLcMl8av7NH5nwNDexw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-languages-service-override/-/monaco-vscode-languages-service-override-21.6.0.tgz", + "integrity": "sha512-sHmraWmJcj//q23SI8T9L5uBgG5jzWhDwz9eHcCFKkNfNnSFbC5hjx9Zi9GWXQyuZ2eRC36kkOSHUYV/c4zVIA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-files-service-override": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-files-service-override": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-layout-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-layout-service-override/-/monaco-vscode-layout-service-override-20.2.1.tgz", - "integrity": "sha512-P2857Tn7ZX5Xi/DgbxfMjJpXiBwI7+ACmUoQTAPBnKam40ZG7EdNS31s02BUVUx22JmDzBbVJ3vYDCCKaeRJHw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-layout-service-override/-/monaco-vscode-layout-service-override-21.6.0.tgz", + "integrity": "sha512-+0aWhYKS1bT+sgsBryV+B7Q4PVnPNvK4y46VaawcdjMRrpZx3BtO4+00VBY4E4W1kYDh26NS2XSGV0ot07s9kw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-6bf85d7b-e6e3-54e9-9bc1-7e08d663f0f6-common": "20.2.1", - "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-6bf85d7b-e6e3-54e9-9bc1-7e08d663f0f6-common": "21.6.0", + "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-localization-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-localization-service-override/-/monaco-vscode-localization-service-override-20.2.1.tgz", - "integrity": "sha512-DHLSu88FWHBSY724GuPIA/sLdIw/j0Bdj3LwdzN46T5R+VV3H7sqtqTLBbs+aY3r9GweNUus3GkwjWZXXGNq/Q==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-localization-service-override/-/monaco-vscode-localization-service-override-21.6.0.tgz", + "integrity": "sha512-rVOTG0FS4S2rSK57vXC5nQ+Gp1ejtNlwUu3ak+Spr0wtlFJm6/eAKY5lkvg2ls8kGj+LEt9aI2tUDAuVESbkTQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-a654b07e-8806-5425-b124-18f03ba8e11a-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-a654b07e-8806-5425-b124-18f03ba8e11a-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-log-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-log-service-override/-/monaco-vscode-log-service-override-20.2.1.tgz", - "integrity": "sha512-CUTxmqqFeu8Sbw24ot6S3wSQDE5DOTMHrK3gC/heNjkWDPXQTu10QteWnBekfP5Czbm+huXUDljL41BVHoD5rQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-log-service-override/-/monaco-vscode-log-service-override-21.6.0.tgz", + "integrity": "sha512-mjDH/ixg5x75s6NF0vi6tzWWKlm5jovLJc+O3i/rhoryhFeip/04WHtq9mfuceY7iQiH7x2BuhiRcq5OSNbYBw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-abed5a84-8a82-5f84-9412-88a736235bae-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-cea4d01f-6526-5c2f-8b09-b168fead499f-common": "20.2.1", - "@codingame/monaco-vscode-environment-service-override": "20.2.1" + "@codingame/monaco-vscode-abed5a84-8a82-5f84-9412-88a736235bae-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-cea4d01f-6526-5c2f-8b09-b168fead499f-common": "21.6.0", + "@codingame/monaco-vscode-environment-service-override": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-model-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-model-service-override/-/monaco-vscode-model-service-override-20.2.1.tgz", - "integrity": "sha512-ff/hBbV1ERU6dcoCJSkBZyg1BvJ8JOmmDt8rmoaWMNoub3hdsmvA87jKw1iblhv/EtKYL0R9J25+6e9LIU2FAQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-model-service-override/-/monaco-vscode-model-service-override-21.6.0.tgz", + "integrity": "sha512-RuL6r1HrCHSUX8G19VXvk5P5M79wgpMB4L1+OMhNkOlwcw4e+Aonh4xM/eV8DqC2N7RMWuc7hTWUPs/BdwLh7A==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-0c06bfba-d24d-5c4d-90cd-b40cefb7f811-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1" + "@codingame/monaco-vscode-0c06bfba-d24d-5c4d-90cd-b40cefb7f811-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-monarch-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-monarch-service-override/-/monaco-vscode-monarch-service-override-20.2.1.tgz", - "integrity": "sha512-HX3VIvV+Y/tW592vs/KogAXXFHDm2U4bgctERjXlgyr1c4x9Gz7FXowDVS5g81Sp19uBY9WDKsHAS8zcrbJAgQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-monarch-service-override/-/monaco-vscode-monarch-service-override-21.6.0.tgz", + "integrity": "sha512-MAbz8i5dkIhAap6KPSMbHXkcVm/8bkRnGxsK7WametT8FVCM1sZak7VbhJu6jAXAPdIQYX/ATTovAycIYZ4zxw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-quickaccess-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-quickaccess-service-override/-/monaco-vscode-quickaccess-service-override-20.2.1.tgz", - "integrity": "sha512-bx5dbe60EPOBesHcC0kJySmmix72vQvRLf2cBl7J3Xncs4yaShlaaScgwL5BwmlNIdpMD8Ay0vN1AOT38PKrkQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-quickaccess-service-override/-/monaco-vscode-quickaccess-service-override-21.6.0.tgz", + "integrity": "sha512-ELUuZ86ogmcTLLX2mKc5zl3yv1L0fPiRwmwTIkq3xwmj+vyNesBXmL1zNaauMvj7MV+P5VPU1LEBptlpZvKwqQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "20.2.1", - "@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common": "20.2.1", - "@codingame/monaco-vscode-40cada32-7e9c-528a-81fc-766e4da54147-common": "20.2.1", - "@codingame/monaco-vscode-9a1a5840-af83-5d07-a156-ba32a36c5c4b-common": "20.2.1", - "@codingame/monaco-vscode-a17e9d37-b6c1-5556-8402-5db73960fae3-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-d609a7d3-bf87-551a-884f-550a8b327ec5-common": "20.2.1", - "@codingame/monaco-vscode-eda30bac-0984-5b42-9362-c68996b85232-common": "20.2.1" + "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "21.6.0", + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-40cada32-7e9c-528a-81fc-766e4da54147-common": "21.6.0", + "@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common": "21.6.0", + "@codingame/monaco-vscode-9a1a5840-af83-5d07-a156-ba32a36c5c4b-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-d609a7d3-bf87-551a-884f-550a8b327ec5-common": "21.6.0", + "@codingame/monaco-vscode-eda30bac-0984-5b42-9362-c68996b85232-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-standalone-css-language-features": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-standalone-css-language-features/-/monaco-vscode-standalone-css-language-features-20.2.1.tgz", - "integrity": "sha512-gd4C4NpPyi20BLBp7pr+52rZlNp9uLyFHJbSPGpH+MoAG0xTOYAEq4w9f4dWV6+ZkL4eLiADAFQO3ZXMQG/pHw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-standalone-css-language-features/-/monaco-vscode-standalone-css-language-features-21.6.0.tgz", + "integrity": "sha512-CQGzycWi1uLY3+IMPRRBrLDwnTUxDMlH81/EE5klh+BtsgJzX6YyHcKr+Mvs6127279jR/b2yYyU9SZEMiheLg==", "license": "MIT", "dependencies": { - "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@20.2.1" + "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@21.6.0" } }, "node_modules/@codingame/monaco-vscode-standalone-html-language-features": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-standalone-html-language-features/-/monaco-vscode-standalone-html-language-features-20.2.1.tgz", - "integrity": "sha512-t8DGn/NUBrZHa+4qQB/KKRW5dCMllKBHIWlkMs+lF+KVFWpWf2alURn0ojsIXH4hmfOvgdW5y/hYlkdhJBEY+Q==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-standalone-html-language-features/-/monaco-vscode-standalone-html-language-features-21.6.0.tgz", + "integrity": "sha512-jCG1YAgSRzwp/eoFYYj+c2A9Uu6tUKUMGiZaLip1RqOgBY2ZoN5ON0ypXkJei5MO4mCqTz6zkXAwIMiilhTN4w==", "license": "MIT", "dependencies": { - "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@20.2.1" + "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@21.6.0" } }, "node_modules/@codingame/monaco-vscode-standalone-json-language-features": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-standalone-json-language-features/-/monaco-vscode-standalone-json-language-features-20.2.1.tgz", - "integrity": "sha512-jgzQ5RYfVCkHCeUksB1Ak16Q9Jkzj2FGWEYepiTaNkHNGm0LwH5q8b/w+vtxlAVaPVnegDGs+c3yiElVPnwViA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-standalone-json-language-features/-/monaco-vscode-standalone-json-language-features-21.6.0.tgz", + "integrity": "sha512-ZaEt3lPJ2cFC1bhUv9LCf9865s2+cqWsG6X/UyhUTdut9Z4ccCIXjrZPRLDRDlHHp82jkGvXFhPK6RxnXYhzwg==", "license": "MIT", "dependencies": { - "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@20.2.1" + "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@21.6.0" } }, "node_modules/@codingame/monaco-vscode-standalone-languages": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-standalone-languages/-/monaco-vscode-standalone-languages-20.2.1.tgz", - "integrity": "sha512-Gzyyaolj2HtwykO1C0/zTr7JFIhe5I3Ed1nk5F53UA6FihL2ttej/K06tRplZTl5GsAJF95NmtYr0F4+n9baKg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-standalone-languages/-/monaco-vscode-standalone-languages-21.6.0.tgz", + "integrity": "sha512-wSsckW8q6BvLucRhPEQZOerAcaKdwMaBtYo+F2wCkA0a0NvGGUBcGnkUgY7T9fIFynRfvDRBi29Wf3NQm0LgvA==", "license": "MIT", "dependencies": { - "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@20.2.1" + "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@21.6.0" } }, "node_modules/@codingame/monaco-vscode-standalone-typescript-language-features": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-standalone-typescript-language-features/-/monaco-vscode-standalone-typescript-language-features-20.2.1.tgz", - "integrity": "sha512-gnRs9Q6gn6zGwPLM6tuTT91dfSCzotbe0QDM8se4OQ4auyJgf0ITvWwQnf0sH92NMVz/Q5RtuQEjSrwnF0VeeA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-standalone-typescript-language-features/-/monaco-vscode-standalone-typescript-language-features-21.6.0.tgz", + "integrity": "sha512-qr08fvnZtYQhCxLkzSq+JCGM6gikv19O/Vj1kUdxLU8TqUDgYuDvRyrcXL83aX2uyQArfyoo4xJRdvvA9ULOsw==", "license": "MIT", "dependencies": { - "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@20.2.1" + "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@21.6.0" } }, "node_modules/@codingame/monaco-vscode-textmate-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-textmate-service-override/-/monaco-vscode-textmate-service-override-20.2.1.tgz", - "integrity": "sha512-cuAnIJudrAoxfhvPONGcjmM9VCcg+yKFT1SzObq3Wrh6tAOaJWzqOYgdlKQgh8parfKy3NE3oRE5C1Fo/HRBdw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-textmate-service-override/-/monaco-vscode-textmate-service-override-21.6.0.tgz", + "integrity": "sha512-BhtfRCYQgr+rR9H4i+WwaaWkaoqnHVmgzuMAh/Q8tUSp5AQdDv5RYlAe7eOV5HxxsRK5t4yUW4b1/6tA0hfMNQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-33833ac7-3af3-5e9d-8fb9-11838d852c59-common": "20.2.1", - "@codingame/monaco-vscode-9a934394-0cf8-512d-939b-77e71f69cebb-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-be143a32-d60a-5489-a1d2-c83ea7eff6bf-common": "20.2.1", - "@codingame/monaco-vscode-files-service-override": "20.2.1", + "@codingame/monaco-vscode-33833ac7-3af3-5e9d-8fb9-11838d852c59-common": "21.6.0", + "@codingame/monaco-vscode-9a934394-0cf8-512d-939b-77e71f69cebb-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-be143a32-d60a-5489-a1d2-c83ea7eff6bf-common": "21.6.0", + "@codingame/monaco-vscode-files-service-override": "21.6.0", "vscode-oniguruma": "1.7.0", "vscode-textmate": "9.2.0" } }, "node_modules/@codingame/monaco-vscode-theme-defaults-default-extension": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-theme-defaults-default-extension/-/monaco-vscode-theme-defaults-default-extension-20.2.1.tgz", - "integrity": "sha512-m0wJAVeUEAcnVgda8Kcw+22FED5/2llQNE7RmrjFF64oSG4qDdn11ptwtfPhovjrCcECHocek8TojnYV+tycYA==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-theme-defaults-default-extension/-/monaco-vscode-theme-defaults-default-extension-21.6.0.tgz", + "integrity": "sha512-64ysP4o00Tyng04TjdBQebEG8KxPDKS6wyoq7lp4A6/BkK41tJhjiq7+0i4wcgIPS8a2m5TIX/VvKcOzYWVyOg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-theme-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-theme-service-override/-/monaco-vscode-theme-service-override-20.2.1.tgz", - "integrity": "sha512-ixlZAMJCKB6up2a0CSSJTr0LzEYY9JzLjunzzdCLcejPrrXrinHiSez3xhYx5cEDGBRC2cbuX3HDmHv2+ZDjWw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-theme-service-override/-/monaco-vscode-theme-service-override-21.6.0.tgz", + "integrity": "sha512-LcY1HTZSY94Q7qnfdDanNyc8pE/6TrUYUoU09iF3FzjAiR8kiDQBsHja5vkrVGt8IYjwnSSxE2SeA/cSHvbfLQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common": "20.2.1", - "@codingame/monaco-vscode-9a934394-0cf8-512d-939b-77e71f69cebb-common": "20.2.1", - "@codingame/monaco-vscode-9d0168a3-519b-57f3-9bcc-89efc41f951a-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-files-service-override": "20.2.1" + "@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common": "21.6.0", + "@codingame/monaco-vscode-9a934394-0cf8-512d-939b-77e71f69cebb-common": "21.6.0", + "@codingame/monaco-vscode-9d0168a3-519b-57f3-9bcc-89efc41f951a-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-files-service-override": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-view-banner-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-view-banner-service-override/-/monaco-vscode-view-banner-service-override-20.2.1.tgz", - "integrity": "sha512-u14wnspdSq87q7hvqWUq1psxBeDfTdq5Gnsc9OpcjgWiBPemA10Rnh/Ep7Oubmo31CR9GfIISglyYeuTj59dYw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-view-banner-service-override/-/monaco-vscode-view-banner-service-override-21.6.0.tgz", + "integrity": "sha512-CLPjwBewnVBJv0bp+90kSjyzLrMV++AFsPn3EQsEnTFtth4144H1Y2NMsgBF/AziOKV7jnVedHMwQNIuWDDhlQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common": "20.2.1", - "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common": "21.6.0", + "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-view-common-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-view-common-service-override/-/monaco-vscode-view-common-service-override-20.2.1.tgz", - "integrity": "sha512-BfVDGZXFbVEJs1aOI0VnyaofDPxgLjILhCYyLz8t14WFH61y7W5QBVfOZpyv1dPPzg1+1EXjSPwHqn6shmN91A==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-view-common-service-override/-/monaco-vscode-view-common-service-override-21.6.0.tgz", + "integrity": "sha512-o19a+pj733J7OTrxBZTGARIq9/+JqLobdEEOfhvNHfRsWyepHyu3eerEe5EadpUlYy4nY7JiesVWcdzGY0vxnw==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-0c06bfba-d24d-5c4d-90cd-b40cefb7f811-common": "20.2.1", - "@codingame/monaco-vscode-0cc5da60-f921-59b9-bd8c-a018e93c0a6f-common": "20.2.1", - "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "20.2.1", - "@codingame/monaco-vscode-1b4486de-4fe4-59c4-9e6d-34f265ff6625-common": "20.2.1", - "@codingame/monaco-vscode-2a94c04a-b85b-5669-b06b-89c1bfa11cb9-common": "20.2.1", - "@codingame/monaco-vscode-3109a756-1f83-5d09-945b-9f0fcad928f0-common": "20.2.1", - "@codingame/monaco-vscode-40cada32-7e9c-528a-81fc-766e4da54147-common": "20.2.1", - "@codingame/monaco-vscode-4a316137-39d1-5d77-8b53-112db3547c1e-common": "20.2.1", - "@codingame/monaco-vscode-501b06ab-3f58-516b-8a1a-c29d375d3da4-common": "20.2.1", - "@codingame/monaco-vscode-523730aa-81e6-55d7-9916-87ad537fe087-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-615ce609-8555-545a-a549-47bd9f80e9f8-common": "20.2.1", - "@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common": "20.2.1", - "@codingame/monaco-vscode-6980eeab-47bb-5a48-8e15-32caf0785565-common": "20.2.1", - "@codingame/monaco-vscode-6db1b967-5327-5c5c-8c17-bd92774c0fb2-common": "20.2.1", - "@codingame/monaco-vscode-72a1b7d3-3f58-5545-9b7e-f579bd003081-common": "20.2.1", - "@codingame/monaco-vscode-7869cfe8-f42c-5721-9f2b-7d04a6a41f16-common": "20.2.1", - "@codingame/monaco-vscode-7f39b6f1-3542-5430-8760-0f404d7a7cee-common": "20.2.1", - "@codingame/monaco-vscode-897bebad-39df-57cb-8a57-36a271d038be-common": "20.2.1", - "@codingame/monaco-vscode-89a82baf-8ded-5b2f-b8af-e5fbd72dc5ad-common": "20.2.1", - "@codingame/monaco-vscode-8ccb7637-50ea-5359-97bf-00015d7fe567-common": "20.2.1", - "@codingame/monaco-vscode-9c84f943-bcb5-5bcf-92a6-91f66a732f26-common": "20.2.1", - "@codingame/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common": "20.2.1", - "@codingame/monaco-vscode-a17e9d37-b6c1-5556-8402-5db73960fae3-common": "20.2.1", - "@codingame/monaco-vscode-a654b07e-8806-5425-b124-18f03ba8e11a-common": "20.2.1", - "@codingame/monaco-vscode-a8d3bd74-e63e-5327-96e8-4f931661e329-common": "20.2.1", - "@codingame/monaco-vscode-ac93482b-2178-52df-a200-ba0d1a4963fb-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-bc6d9a89-1625-5010-b57e-ff44151144fe-common": "20.2.1", - "@codingame/monaco-vscode-bulk-edit-service-override": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1", - "@codingame/monaco-vscode-d26a96d3-122c-5a3d-a04d-deb5ff0f19c0-common": "20.2.1", - "@codingame/monaco-vscode-d481a59e-259c-524e-bee1-76483d75d3a1-common": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1", - "@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common": "20.2.1", - "@codingame/monaco-vscode-e59ecb8c-db32-5324-8fe4-cf9921fd92b8-common": "20.2.1", - "@codingame/monaco-vscode-e72c94ca-257a-5b75-8b68-5a5fa3c18255-common": "20.2.1", - "@codingame/monaco-vscode-f1bbc6d3-6129-583c-a2ba-c80b832993d2-common": "20.2.1", - "@codingame/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common": "20.2.1", - "@codingame/monaco-vscode-ff9fa663-eae3-5274-8573-c2b918871e4b-common": "20.2.1" + "@codingame/monaco-vscode-0c06bfba-d24d-5c4d-90cd-b40cefb7f811-common": "21.6.0", + "@codingame/monaco-vscode-0cc5da60-f921-59b9-bd8c-a018e93c0a6f-common": "21.6.0", + "@codingame/monaco-vscode-158b9837-fc78-5d9c-86f5-9134e4358643-common": "21.6.0", + "@codingame/monaco-vscode-1b4486de-4fe4-59c4-9e6d-34f265ff6625-common": "21.6.0", + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-2a94c04a-b85b-5669-b06b-89c1bfa11cb9-common": "21.6.0", + "@codingame/monaco-vscode-3109a756-1f83-5d09-945b-9f0fcad928f0-common": "21.6.0", + "@codingame/monaco-vscode-40cada32-7e9c-528a-81fc-766e4da54147-common": "21.6.0", + "@codingame/monaco-vscode-4a316137-39d1-5d77-8b53-112db3547c1e-common": "21.6.0", + "@codingame/monaco-vscode-4fad3647-b95d-5c19-bab1-bb9de627a5ec-common": "21.6.0", + "@codingame/monaco-vscode-523730aa-81e6-55d7-9916-87ad537fe087-common": "21.6.0", + "@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common": "21.6.0", + "@codingame/monaco-vscode-615ce609-8555-545a-a549-47bd9f80e9f8-common": "21.6.0", + "@codingame/monaco-vscode-670aae94-7f88-54d7-90ea-6fcbef423557-common": "21.6.0", + "@codingame/monaco-vscode-6980eeab-47bb-5a48-8e15-32caf0785565-common": "21.6.0", + "@codingame/monaco-vscode-6db1b967-5327-5c5c-8c17-bd92774c0fb2-common": "21.6.0", + "@codingame/monaco-vscode-71c8dbff-4c98-552f-aef0-e72b00fdcfc0-common": "21.6.0", + "@codingame/monaco-vscode-72a1b7d3-3f58-5545-9b7e-f579bd003081-common": "21.6.0", + "@codingame/monaco-vscode-7869cfe8-f42c-5721-9f2b-7d04a6a41f16-common": "21.6.0", + "@codingame/monaco-vscode-7f39b6f1-3542-5430-8760-0f404d7a7cee-common": "21.6.0", + "@codingame/monaco-vscode-897bebad-39df-57cb-8a57-36a271d038be-common": "21.6.0", + "@codingame/monaco-vscode-89a82baf-8ded-5b2f-b8af-e5fbd72dc5ad-common": "21.6.0", + "@codingame/monaco-vscode-8ccb7637-50ea-5359-97bf-00015d7fe567-common": "21.6.0", + "@codingame/monaco-vscode-9a5ab9e7-d838-5831-9eb4-e79ea3764dcb-common": "21.6.0", + "@codingame/monaco-vscode-9c84f943-bcb5-5bcf-92a6-91f66a732f26-common": "21.6.0", + "@codingame/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common": "21.6.0", + "@codingame/monaco-vscode-a654b07e-8806-5425-b124-18f03ba8e11a-common": "21.6.0", + "@codingame/monaco-vscode-ac93482b-2178-52df-a200-ba0d1a4963fb-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-bc6d9a89-1625-5010-b57e-ff44151144fe-common": "21.6.0", + "@codingame/monaco-vscode-bulk-edit-service-override": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0", + "@codingame/monaco-vscode-d26a96d3-122c-5a3d-a04d-deb5ff0f19c0-common": "21.6.0", + "@codingame/monaco-vscode-d481a59e-259c-524e-bee1-76483d75d3a1-common": "21.6.0", + "@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common": "21.6.0", + "@codingame/monaco-vscode-e59ecb8c-db32-5324-8fe4-cf9921fd92b8-common": "21.6.0", + "@codingame/monaco-vscode-e72c94ca-257a-5b75-8b68-5a5fa3c18255-common": "21.6.0", + "@codingame/monaco-vscode-f1bbc6d3-6129-583c-a2ba-c80b832993d2-common": "21.6.0", + "@codingame/monaco-vscode-f24e325c-2ce0-5bba-8236-bfc4f53180ab-common": "21.6.0", + "@codingame/monaco-vscode-ff9fa663-eae3-5274-8573-c2b918871e4b-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-view-status-bar-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-view-status-bar-service-override/-/monaco-vscode-view-status-bar-service-override-20.2.1.tgz", - "integrity": "sha512-aBflohzbcYW6L8ZZ1lZpq+oRxU67z5zj1gxGJrTyV8QFTreNacHvzkmPF0eVnO11wnsOxBFCP2/FVaYbcheJJQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-view-status-bar-service-override/-/monaco-vscode-view-status-bar-service-override-21.6.0.tgz", + "integrity": "sha512-5a7xuQM+WGaU0Dw5vfLCi6SiYQKnuJbAbuLlp6u723I25e1IJ0Glt+bXYdyKxpaYT5FVx4xJOBFxLuedGNyI8g==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-0cc5da60-f921-59b9-bd8c-a018e93c0a6f-common": "20.2.1", - "@codingame/monaco-vscode-622c0cca-d5fa-59b6-b730-0715afcf93ee-common": "20.2.1", - "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" + "@codingame/monaco-vscode-0cc5da60-f921-59b9-bd8c-a018e93c0a6f-common": "21.6.0", + "@codingame/monaco-vscode-622c0cca-d5fa-59b6-b730-0715afcf93ee-common": "21.6.0", + "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-view-title-bar-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-view-title-bar-service-override/-/monaco-vscode-view-title-bar-service-override-20.2.1.tgz", - "integrity": "sha512-qct4MerxxRJu629OftQsDEyovCDR/EORZwn9aJNXvpL4M/Xi5oNQaQvg8ZzNuUCiqwfnmlSI8b5Jsp6zVCtkEg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-view-title-bar-service-override/-/monaco-vscode-view-title-bar-service-override-21.6.0.tgz", + "integrity": "sha512-smoaqHc50wUEc3kI5sEEgDhjFJkh7WXq/aiAUUfkRsy3fA8fs8LmEZ5z5ueI7zx1lIvuT89HE5xq9+jw96NqjA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-08d1b4da-daf2-5f0d-8c50-ca6a6986c50f-common": "20.2.1", - "@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common": "20.2.1", - "@codingame/monaco-vscode-40cada32-7e9c-528a-81fc-766e4da54147-common": "20.2.1", - "@codingame/monaco-vscode-60014c9d-b815-501d-83a9-4b08725c2ec2-common": "20.2.1", - "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "20.2.1", - "@codingame/monaco-vscode-96e83782-7f38-572e-8787-02e981f1c54f-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common": "20.2.1", - "@codingame/monaco-vscode-ebba7d85-8a22-5735-adf4-8299cd976dce-common": "20.2.1" + "@codingame/monaco-vscode-08d1b4da-daf2-5f0d-8c50-ca6a6986c50f-common": "21.6.0", + "@codingame/monaco-vscode-40cada32-7e9c-528a-81fc-766e4da54147-common": "21.6.0", + "@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common": "21.6.0", + "@codingame/monaco-vscode-5e2c24a1-3217-55e8-bc90-521eaf7df5a6-common": "21.6.0", + "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "21.6.0", + "@codingame/monaco-vscode-96e83782-7f38-572e-8787-02e981f1c54f-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-dbfe5f85-b426-55ed-a79b-5f811b395762-common": "21.6.0", + "@codingame/monaco-vscode-ebba7d85-8a22-5735-adf4-8299cd976dce-common": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-views-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-views-service-override/-/monaco-vscode-views-service-override-20.2.1.tgz", - "integrity": "sha512-TephHnUri9ZJqIIIvesBbHcY9UalEHdfNCpDAA5bmjlPS9ZgrkJ+kf1vmRF5+i38rTbUZmAuz/EQbgs6OKq+ag==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-views-service-override/-/monaco-vscode-views-service-override-21.6.0.tgz", + "integrity": "sha512-7+Zjq7xC1QtO1LZMdsbiMmd0RgomjAOrljvu/66EAE9dJ4rW1dXU+IbViEQJpRYiYrnNBfgXT+DQ/d7Yt9C3Vg==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-6980eeab-47bb-5a48-8e15-32caf0785565-common": "20.2.1", - "@codingame/monaco-vscode-6bf85d7b-e6e3-54e9-9bc1-7e08d663f0f6-common": "20.2.1", - "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "20.2.1", - "@codingame/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common": "20.2.1", - "@codingame/monaco-vscode-a8d3bd74-e63e-5327-96e8-4f931661e329-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1", - "@codingame/monaco-vscode-keybindings-service-override": "20.2.1", - "@codingame/monaco-vscode-layout-service-override": "20.2.1", - "@codingame/monaco-vscode-quickaccess-service-override": "20.2.1", - "@codingame/monaco-vscode-view-common-service-override": "20.2.1" + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-6980eeab-47bb-5a48-8e15-32caf0785565-common": "21.6.0", + "@codingame/monaco-vscode-6bf85d7b-e6e3-54e9-9bc1-7e08d663f0f6-common": "21.6.0", + "@codingame/monaco-vscode-71c8dbff-4c98-552f-aef0-e72b00fdcfc0-common": "21.6.0", + "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "21.6.0", + "@codingame/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0", + "@codingame/monaco-vscode-keybindings-service-override": "21.6.0", + "@codingame/monaco-vscode-layout-service-override": "21.6.0", + "@codingame/monaco-vscode-quickaccess-service-override": "21.6.0", + "@codingame/monaco-vscode-view-common-service-override": "21.6.0" } }, "node_modules/@codingame/monaco-vscode-workbench-service-override": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-workbench-service-override/-/monaco-vscode-workbench-service-override-20.2.1.tgz", - "integrity": "sha512-gXQvdA/NJbNpc/4+3cd05vAd6S0xGr4AGDD1rn4AKIBRg5BI3tksBtUUX0CXxw62Ogxba36j43QXnf8QeMbJOg==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-workbench-service-override/-/monaco-vscode-workbench-service-override-21.6.0.tgz", + "integrity": "sha512-vvUzaLAbOLj16SjV4T+OJXRwF0uIM9Q7jpYYoKXCkN+Y8CJpGCDUSwCsLGx4Tz+f2H/ZzUWsv5Wr8xlznD18DA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-1021b67c-93e5-5c78-a270-cbdb2574d980-common": "20.2.1", - "@codingame/monaco-vscode-256d5b78-0649-50e9-8354-2807f95f68f4-common": "20.2.1", - "@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common": "20.2.1", - "@codingame/monaco-vscode-494be54c-bd37-5b3c-af70-02f086e28768-common": "20.2.1", - "@codingame/monaco-vscode-6980eeab-47bb-5a48-8e15-32caf0785565-common": "20.2.1", - "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "20.2.1", - "@codingame/monaco-vscode-9c84f943-bcb5-5bcf-92a6-91f66a732f26-common": "20.2.1", - "@codingame/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common": "20.2.1", - "@codingame/monaco-vscode-a8d3bd74-e63e-5327-96e8-4f931661e329-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "20.2.1", - "@codingame/monaco-vscode-d941ac7b-412f-57e3-b1bf-f6b0eb253b21-common": "20.2.1", - "@codingame/monaco-vscode-f6f55824-df83-5ffc-ac26-50fd4df4fe0e-common": "20.2.1", - "@codingame/monaco-vscode-keybindings-service-override": "20.2.1", - "@codingame/monaco-vscode-quickaccess-service-override": "20.2.1", - "@codingame/monaco-vscode-view-banner-service-override": "20.2.1", - "@codingame/monaco-vscode-view-common-service-override": "20.2.1", - "@codingame/monaco-vscode-view-status-bar-service-override": "20.2.1", - "@codingame/monaco-vscode-view-title-bar-service-override": "20.2.1" + "@codingame/monaco-vscode-1021b67c-93e5-5c78-a270-cbdb2574d980-common": "21.6.0", + "@codingame/monaco-vscode-23b6fb38-5e58-5886-b34b-27abc4f5df02-common": "21.6.0", + "@codingame/monaco-vscode-256d5b78-0649-50e9-8354-2807f95f68f4-common": "21.6.0", + "@codingame/monaco-vscode-494be54c-bd37-5b3c-af70-02f086e28768-common": "21.6.0", + "@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common": "21.6.0", + "@codingame/monaco-vscode-6980eeab-47bb-5a48-8e15-32caf0785565-common": "21.6.0", + "@codingame/monaco-vscode-71c8dbff-4c98-552f-aef0-e72b00fdcfc0-common": "21.6.0", + "@codingame/monaco-vscode-85886bdb-61c5-52f1-8eb7-d1d32f6f8cbd-common": "21.6.0", + "@codingame/monaco-vscode-9c84f943-bcb5-5bcf-92a6-91f66a732f26-common": "21.6.0", + "@codingame/monaco-vscode-9efc1f50-c7de-55d6-8b28-bcc88bd49b5a-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-caeb744c-8e3f-5c11-80fb-0f057d24d544-common": "21.6.0", + "@codingame/monaco-vscode-keybindings-service-override": "21.6.0", + "@codingame/monaco-vscode-quickaccess-service-override": "21.6.0", + "@codingame/monaco-vscode-view-banner-service-override": "21.6.0", + "@codingame/monaco-vscode-view-common-service-override": "21.6.0", + "@codingame/monaco-vscode-view-status-bar-service-override": "21.6.0", + "@codingame/monaco-vscode-view-title-bar-service-override": "21.6.0" } }, "node_modules/@csstools/css-parser-algorithms": { @@ -1847,446 +1863,38 @@ "@csstools/css-tokenizer": "^2.4.1" } }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.9.tgz", - "integrity": "sha512-OaGtL73Jck6pBKjNIe24BnFE6agGl+6KxDtTfHhy1HmhthfKouEcOhqpSL64K4/0WCtbKFLOdzD/44cJ4k9opA==", - "cpu": [ - "ppc64" - ], + "node_modules/@emnapi/core": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.5.0.tgz", + "integrity": "sha512-sbP8GzB1WDzacS8fgNPpHlp6C9VZe+SJP3F90W9rLemaQj2PzIuTEl1qDOYQf58YIpyjViI24y9aPWCjEzY2cg==", "dev": true, "license": "MIT", "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" + "dependencies": { + "@emnapi/wasi-threads": "1.1.0", + "tslib": "^2.4.0" } }, - "node_modules/@esbuild/android-arm": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.9.tgz", - "integrity": "sha512-5WNI1DaMtxQ7t7B6xa572XMXpHAaI/9Hnhk8lcxF4zVN4xstUgTlvuGDorBguKEnZO70qwEcLpfifMLoxiPqHQ==", - "cpu": [ - "arm" - ], + "node_modules/@emnapi/runtime": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.5.0.tgz", + "integrity": "sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==", "dev": true, "license": "MIT", "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" + "dependencies": { + "tslib": "^2.4.0" } }, - "node_modules/@esbuild/android-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.9.tgz", - "integrity": "sha512-IDrddSmpSv51ftWslJMvl3Q2ZT98fUSL2/rlUXuVqRXHCs5EUF1/f+jbjF5+NG9UffUDMCiTyh8iec7u8RlTLg==", - "cpu": [ - "arm64" - ], + "node_modules/@emnapi/wasi-threads": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", + "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", "dev": true, "license": "MIT", "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.9.tgz", - "integrity": "sha512-I853iMZ1hWZdNllhVZKm34f4wErd4lMyeV7BLzEExGEIZYsOzqDWDf+y082izYUE8gtJnYHdeDpN/6tUdwvfiw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.9.tgz", - "integrity": "sha512-XIpIDMAjOELi/9PB30vEbVMs3GV1v2zkkPnuyRRURbhqjyzIINwj+nbQATh4H9GxUgH1kFsEyQMxwiLFKUS6Rg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.9.tgz", - "integrity": "sha512-jhHfBzjYTA1IQu8VyrjCX4ApJDnH+ez+IYVEoJHeqJm9VhG9Dh2BYaJritkYK3vMaXrf7Ogr/0MQ8/MeIefsPQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.9.tgz", - "integrity": "sha512-z93DmbnY6fX9+KdD4Ue/H6sYs+bhFQJNCPZsi4XWJoYblUqT06MQUdBCpcSfuiN72AbqeBFu5LVQTjfXDE2A6Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.9.tgz", - "integrity": "sha512-mrKX6H/vOyo5v71YfXWJxLVxgy1kyt1MQaD8wZJgJfG4gq4DpQGpgTB74e5yBeQdyMTbgxp0YtNj7NuHN0PoZg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.9.tgz", - "integrity": "sha512-HBU2Xv78SMgaydBmdor38lg8YDnFKSARg1Q6AT0/y2ezUAKiZvc211RDFHlEZRFNRVhcMamiToo7bDx3VEOYQw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.9.tgz", - "integrity": "sha512-BlB7bIcLT3G26urh5Dmse7fiLmLXnRlopw4s8DalgZ8ef79Jj4aUcYbk90g8iCa2467HX8SAIidbL7gsqXHdRw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.9.tgz", - "integrity": "sha512-e7S3MOJPZGp2QW6AK6+Ly81rC7oOSerQ+P8L0ta4FhVi+/j/v2yZzx5CqqDaWjtPFfYz21Vi1S0auHrap3Ma3A==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.9.tgz", - "integrity": "sha512-Sbe10Bnn0oUAB2AalYztvGcK+o6YFFA/9829PhOCUS9vkJElXGdphz0A3DbMdP8gmKkqPmPcMJmJOrI3VYB1JQ==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.9.tgz", - "integrity": "sha512-YcM5br0mVyZw2jcQeLIkhWtKPeVfAerES5PvOzaDxVtIyZ2NUBZKNLjC5z3/fUlDgT6w89VsxP2qzNipOaaDyA==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.9.tgz", - "integrity": "sha512-++0HQvasdo20JytyDpFvQtNrEsAgNG2CY1CLMwGXfFTKGBGQT3bOeLSYE2l1fYdvML5KUuwn9Z8L1EWe2tzs1w==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.9.tgz", - "integrity": "sha512-uNIBa279Y3fkjV+2cUjx36xkx7eSjb8IvnL01eXUKXez/CBHNRw5ekCGMPM0BcmqBxBcdgUWuUXmVWwm4CH9kg==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.9.tgz", - "integrity": "sha512-Mfiphvp3MjC/lctb+7D287Xw1DGzqJPb/J2aHHcHxflUo+8tmN/6d4k6I2yFR7BVo5/g7x2Monq4+Yew0EHRIA==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.9.tgz", - "integrity": "sha512-iSwByxzRe48YVkmpbgoxVzn76BXjlYFXC7NvLYq+b+kDjyyk30J0JY47DIn8z1MO3K0oSl9fZoRmZPQI4Hklzg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.9.tgz", - "integrity": "sha512-9jNJl6FqaUG+COdQMjSCGW4QiMHH88xWbvZ+kRVblZsWrkXlABuGdFJ1E9L7HK+T0Yqd4akKNa/lO0+jDxQD4Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.9.tgz", - "integrity": "sha512-RLLdkflmqRG8KanPGOU7Rpg829ZHu8nFy5Pqdi9U01VYtG9Y0zOG6Vr2z4/S+/3zIyOxiK6cCeYNWOFR9QP87g==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.9.tgz", - "integrity": "sha512-YaFBlPGeDasft5IIM+CQAhJAqS3St3nJzDEgsgFixcfZeyGPCd6eJBWzke5piZuZ7CtL656eOSYKk4Ls2C0FRQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.9.tgz", - "integrity": "sha512-1MkgTCuvMGWuqVtAvkpkXFmtL8XhWy+j4jaSO2wxfJtilVCi0ZE37b8uOdMItIHz4I6z1bWWtEX4CJwcKYLcuA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.9.tgz", - "integrity": "sha512-4Xd0xNiMVXKh6Fa7HEJQbrpP3m3DDn43jKxMjxLLRjWnRsfxjORYJlXPO4JNcXtOyfajXorRKY9NkOpTHptErg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.9.tgz", - "integrity": "sha512-WjH4s6hzo00nNezhp3wFIAfmGZ8U7KtrJNlFMRKxiI9mxEK1scOMAaa9i4crUtu+tBr+0IN6JCuAcSBJZfnphw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.9.tgz", - "integrity": "sha512-mGFrVJHmZiRqmP8xFOc6b84/7xa5y5YvR1x8djzXpJBSv/UsNK6aqec+6JDjConTgvvQefdGhFDAs2DLAds6gQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.9.tgz", - "integrity": "sha512-b33gLVU2k11nVx1OhX3C8QQP6UHQK4ZtN56oFWvVXvz2VkDoe6fbG8TOgHFxEvqeqohmRnIHe5A1+HADk4OQww==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.9.tgz", - "integrity": "sha512-PPOl1mi6lpLNQxnGoyAfschAodRFYXJ+9fs6WHXz7CSWKbOqiMZsubC+BQsVKuul+3vKLuwTHsS2c2y9EoKwxQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" + "dependencies": { + "tslib": "^2.4.0" } }, "node_modules/@eslint-community/eslint-utils": { @@ -2417,6 +2025,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@glideapps/ts-necessities": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/@glideapps/ts-necessities/-/ts-necessities-2.2.3.tgz", + "integrity": "sha512-gXi0awOZLHk3TbW55GZLCPP6O+y/b5X1pBXKBVckFONSwF1z1E5ND2BGJsghQFah+pW7pkkyFb2VhUQI2qhL5w==", + "license": "MIT" + }, "node_modules/@hey-api/openapi-ts": { "version": "0.43.2", "resolved": "https://registry.npmjs.org/@hey-api/openapi-ts/-/openapi-ts-0.43.2.tgz", @@ -2492,9 +2106,9 @@ "license": "BSD-3-Clause" }, "node_modules/@internationalized/date": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.9.0.tgz", - "integrity": "sha512-yaN3brAnHRD+4KyyOsJyk49XUvj2wtbNACSqg0bz3u8t2VuzhC8Q5dfRnrSxjnnbDb+ienBnkn1TzQfE154vyg==", + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.10.0.tgz", + "integrity": "sha512-oxDR/NTEJ1k+UFVQElaNIk65E/Z83HK1z1WI3lQyhTtnNg4R5oVXaPzK3jcpKG8UHKDVuDQHzn+wsxSz8RP3aw==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -2762,6 +2376,19 @@ "svelte": "^3.0.0 || ^4.0.0 || ^5.0.0-next.118" } }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.0.7.tgz", + "integrity": "sha512-SeDnOO0Tk7Okiq6DbXmmBODgOAb9dp9gjlphokTUxmt8U3liIP1ZsozBahH69j/RJv+Rfs6IwUKHTgQYJ/HBAw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.5.0", + "@emnapi/runtime": "^1.5.0", + "@tybys/wasm-util": "^0.10.1" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -2800,10 +2427,30 @@ "node": ">= 8" } }, + "node_modules/@oxc-project/runtime": { + "version": "0.92.0", + "resolved": "https://registry.npmjs.org/@oxc-project/runtime/-/runtime-0.92.0.tgz", + "integrity": "sha512-Z7x2dZOmznihvdvCvLKMl+nswtOSVxS2H2ocar+U9xx6iMfTp0VGIrX6a4xB1v80IwOPC7dT1LXIJrY70Xu3Jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.94.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.94.0.tgz", + "integrity": "sha512-+UgQT/4o59cZfH6Cp7G0hwmqEQ0wE+AdIwhikdwnhWI9Dp8CgSY081+Q3O67/wq3VJu8mgUEB93J9EHHn70fOw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, "node_modules/@petamoriken/float16": { - "version": "3.9.2", - "resolved": "https://registry.npmjs.org/@petamoriken/float16/-/float16-3.9.2.tgz", - "integrity": "sha512-VgffxawQde93xKxT3qap3OH+meZf7VaSB5Sqd4Rqc+FP5alWbpOyan/7tRbOAvynjpG3GpdtAuGU/NdhQpmrog==", + "version": "3.9.3", + "resolved": "https://registry.npmjs.org/@petamoriken/float16/-/float16-3.9.3.tgz", + "integrity": "sha512-8awtpHXCx/bNpFt4mt2xdkgtgVvKqty8VbjHI/WWWQuEw+KLzFot3f4+LkQY9YmOtq7A5GdOnqoIC8Pdygjk2g==", "license": "MIT" }, "node_modules/@pkgjs/parseargs": { @@ -2818,13 +2465,13 @@ } }, "node_modules/@playwright/test": { - "version": "1.55.0", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.55.0.tgz", - "integrity": "sha512-04IXzPwHrW69XusN/SIdDdKZBzMfOT9UNT/YiJit/xpy2VuAoB8NHc8Aplb96zsWDddLnbkPL3TsmrS04ZU2xQ==", + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.56.0.tgz", + "integrity": "sha512-Tzh95Twig7hUwwNe381/K3PggZBZblKUe2wv25oIpzWLr6Z0m4KgV1ZVIjnR6GM9ANEqjZD7XsZEa6JL/7YEgg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright": "1.55.0" + "playwright": "1.56.0" }, "bin": { "playwright": "cli.js" @@ -2856,24 +2503,10 @@ "integrity": "sha512-dSMyuNPN2k+tFeNZ0+QJ7S1zDJ0UeNL+lpnPFR9K5avj2V4uG4m6FdjrApQ9Zi35AIocaDp/KGfBD9gR5MLUbQ==", "license": "SEE LICENSE IN LICENSE" }, - "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.50.1.tgz", - "integrity": "sha512-HJXwzoZN4eYTdD8bVV22DN8gsPCAj3V20NHKOs8ezfXanGpmVPR7kalUHd+Y31IJp9stdB87VKPFbsGY3H/2ag==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ] - }, - "node_modules/@rollup/rollup-android-arm64": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.50.1.tgz", - "integrity": "sha512-PZlsJVcjHfcH53mOImyt3bc97Ep3FJDXRpk9sMdGX0qgLmY0EIWxCag6EigerGhLVuL8lDVYNnSo8qnTElO4xw==", + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-beta.42.tgz", + "integrity": "sha512-W5ZKF3TP3bOWuBfotAGp+UGjxOkGV7jRmIRbBA7NFjggx7Oi6vOmGDqpHEIX7kDCiry1cnIsWQaxNvWbMdkvzQ==", "cpu": [ "arm64" ], @@ -2882,12 +2515,15 @@ "optional": true, "os": [ "android" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.50.1.tgz", - "integrity": "sha512-xc6i2AuWh++oGi4ylOFPmzJOEeAa2lJeGUGb4MudOtgfyyjr4UPNK+eEWTPLvmPJIY/pgw6ssFIox23SyrkkJw==", + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-beta.42.tgz", + "integrity": "sha512-abw/wtgJA8OCgaTlL+xJxnN/Z01BwV1rfzIp5Hh9x+IIO6xOBfPsQ0nzi0+rWx3TyZ9FZXyC7bbC+5NpQ9EaXQ==", "cpu": [ "arm64" ], @@ -2896,12 +2532,15 @@ "optional": true, "os": [ "darwin" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.50.1.tgz", - "integrity": "sha512-2ofU89lEpDYhdLAbRdeyz/kX3Y2lpYc6ShRnDjY35bZhd2ipuDMDi6ZTQ9NIag94K28nFMofdnKeHR7BT0CATw==", + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-beta.42.tgz", + "integrity": "sha512-Y/UrZIRVr8CvXVEB88t6PeC46r1K9/QdPEo2ASE/b/KBEyXIx+QbM6kv9QfQVWU2Atly2+SVsQzxQsIvuk3lZQ==", "cpu": [ "x64" ], @@ -2910,26 +2549,15 @@ "optional": true, "os": [ "darwin" - ] - }, - "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.50.1.tgz", - "integrity": "sha512-wOsE6H2u6PxsHY/BeFHA4VGQN3KUJFZp7QJBmDYI983fgxq5Th8FDkVuERb2l9vDMs1D5XhOrhBrnqcEY6l8ZA==", - "cpu": [ - "arm64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.50.1.tgz", - "integrity": "sha512-A/xeqaHTlKbQggxCqispFAcNjycpUEHP52mwMQZUNqDUJFFYtPHCXS1VAG29uMlDzIVr+i00tSFWFLivMcoIBQ==", + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-beta.42.tgz", + "integrity": "sha512-zRM0oOk7BZiy6DoWBvdV4hyEg+j6+WcBZIMHVirMEZRu8hd18kZdJkg+bjVMfCEhwpWeFUfBfZ1qcaZ5UdYzlQ==", "cpu": [ "x64" ], @@ -2938,12 +2566,15 @@ "optional": true, "os": [ "freebsd" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.50.1.tgz", - "integrity": "sha512-54v4okehwl5TaSIkpp97rAHGp7t3ghinRd/vyC1iXqXMfjYUTm7TfYmCzXDoHUPTTf36L8pr0E7YsD3CfB3ZDg==", + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-beta.42.tgz", + "integrity": "sha512-6RjFaC52QNwo7ilU8C5H7swbGlgfTkG9pudXwzr3VYyT18s0C9gLg3mvc7OMPIGqNxnQ0M5lU8j6aQCk2DTRVg==", "cpu": [ "arm" ], @@ -2952,26 +2583,15 @@ "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.50.1.tgz", - "integrity": "sha512-p/LaFyajPN/0PUHjv8TNyxLiA7RwmDoVY3flXHPSzqrGcIp/c2FjwPPP5++u87DGHtw+5kSH5bCJz0mvXngYxw==", - "cpu": [ - "arm" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.50.1.tgz", - "integrity": "sha512-2AbMhFFkTo6Ptna1zO7kAXXDLi7H9fGTbVaIq2AAYO7yzcAsuTNWPHhb2aTA6GPiP+JXh85Y8CiS54iZoj4opw==", + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-beta.42.tgz", + "integrity": "sha512-LMYHM5Sf6ROq+VUwHMDVX2IAuEsWTv4SnlFEedBnMGpvRuQ14lCmD4m5Q8sjyAQCgyha9oghdGoK8AEg1sXZKg==", "cpu": [ "arm64" ], @@ -2980,12 +2600,15 @@ "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.50.1.tgz", - "integrity": "sha512-Cgef+5aZwuvesQNw9eX7g19FfKX5/pQRIyhoXLCiBOrWopjo7ycfB292TX9MDcDijiuIJlx1IzJz3IoCPfqs9w==", + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-beta.42.tgz", + "integrity": "sha512-/bNTYb9aKNhzdbPn3O4MK2aLv55AlrkUKPE4KNfBYjkoZUfDr4jWp7gsSlvTc5A/99V1RCm9axvt616ZzeXGyA==", "cpu": [ "arm64" ], @@ -2994,95 +2617,15 @@ "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-loongarch64-gnu": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.50.1.tgz", - "integrity": "sha512-RPhTwWMzpYYrHrJAS7CmpdtHNKtt2Ueo+BlLBjfZEhYBhK00OsEqM08/7f+eohiF6poe0YRDDd8nAvwtE/Y62Q==", - "cpu": [ - "loong64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-ppc64-gnu": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.50.1.tgz", - "integrity": "sha512-eSGMVQw9iekut62O7eBdbiccRguuDgiPMsw++BVUg+1K7WjZXHOg/YOT9SWMzPZA+w98G+Fa1VqJgHZOHHnY0Q==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.50.1.tgz", - "integrity": "sha512-S208ojx8a4ciIPrLgazF6AgdcNJzQE4+S9rsmOmDJkusvctii+ZvEuIC4v/xFqzbuP8yDjn73oBlNDgF6YGSXQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.50.1.tgz", - "integrity": "sha512-3Ag8Ls1ggqkGUvSZWYcdgFwriy2lWo+0QlYgEFra/5JGtAd6C5Hw59oojx1DeqcA2Wds2ayRgvJ4qxVTzCHgzg==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.50.1.tgz", - "integrity": "sha512-t9YrKfaxCYe7l7ldFERE1BRg/4TATxIg+YieHQ966jwvo7ddHJxPj9cNFWLAzhkVsbBvNA4qTbPVNsZKBO4NSg==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.50.1.tgz", - "integrity": "sha512-MCgtFB2+SVNuQmmjHf+wfI4CMxy3Tk8XjA5Z//A0AKD7QXUYFMQcns91K6dEHBvZPCnhJSyDWLApk40Iq/H3tA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.50.1.tgz", - "integrity": "sha512-nEvqG+0jeRmqaUMuwzlfMKwcIVffy/9KGbAGyoa26iu6eSngAYQ512bMXuqqPrlTyfqdlB9FVINs93j534UJrg==", + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-beta.42.tgz", + "integrity": "sha512-n/SLa4h342oyeGykZdch7Y3GNCNliRPL4k5wkeZ/5eQZs+c6/ZG1SHCJQoy7bZcmxiMyaXs9HoFmv1PEKrZgWg==", "cpu": [ "x64" ], @@ -3091,12 +2634,32 @@ "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-openharmony-arm64": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.50.1.tgz", - "integrity": "sha512-RDsLm+phmT3MJd9SNxA9MNuEAO/J2fhW8GXk62G/B4G7sLVumNFbRwDL6v5NrESb48k+QMqdGbHgEtfU0LCpbA==", + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-beta.42.tgz", + "integrity": "sha512-4PSd46sFzqpLHSGdaSViAb1mk55sCUMpJg+X8ittXaVocQsV3QLG/uydSH8RyL0ngHX5fy3D70LcCzlB15AgHw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-beta.42.tgz", + "integrity": "sha512-BmWoeJJyeZXmZBcfoxG6J9+rl2G7eO47qdTkAzEegj4n3aC6CBIHOuDcbE8BvhZaEjQR0nh0nJrtEDlt65Q7Sw==", "cpu": [ "arm64" ], @@ -3105,12 +2668,32 @@ "optional": true, "os": [ "openharmony" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.50.1.tgz", - "integrity": "sha512-hpZB/TImk2FlAFAIsoElM3tLzq57uxnGYwplg6WDyAxbYczSi8O2eQ+H2Lx74504rwKtZ3N2g4bCUkiamzS6TQ==", + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-beta.42.tgz", + "integrity": "sha512-2Ft32F7uiDTrGZUKws6CLNTlvTWHC33l4vpXrzUucf9rYtUThAdPCOt89Pmn13tNX6AulxjGEP2R0nZjTSW3eQ==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@napi-rs/wasm-runtime": "^1.0.6" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-beta.42.tgz", + "integrity": "sha512-hC1kShXW/z221eG+WzQMN06KepvPbMBknF0iGR3VMYJLOe9gwnSTfGxFT5hf8XrPv7CEZqTWRd0GQpkSHRbGsw==", "cpu": [ "arm64" ], @@ -3119,12 +2702,15 @@ "optional": true, "os": [ "win32" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.50.1.tgz", - "integrity": "sha512-SXjv8JlbzKM0fTJidX4eVsH+Wmnp0/WcD8gJxIZyR6Gay5Qcsmdbi9zVtnbkGPG8v2vMR1AD06lGWy5FLMcG7A==", + "node_modules/@rolldown/binding-win32-ia32-msvc": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.0.0-beta.42.tgz", + "integrity": "sha512-AICBYromawouGjj+GS33369E8Vwhy6UwhQEhQ5evfS8jPCsyVvoICJatbDGDGH01dwtVGLD5eDFzPicUOVpe4g==", "cpu": [ "ia32" ], @@ -3133,12 +2719,15 @@ "optional": true, "os": [ "win32" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.50.1.tgz", - "integrity": "sha512-StxAO/8ts62KZVRAm4JZYq9+NqNsV7RvimNK+YM7ry//zebEH6meuugqW/P5OFUCjyQgui+9fUxT6d5NShvMvA==", + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-beta.42.tgz", + "integrity": "sha512-XpZ0M+tjoEiSc9c+uZR7FCnOI0uxDRNs1elGOMjeB0pUP1QmvVbZGYNsyLbLoP4u7e3VQN8rie1OQ8/mB6rcJg==", "cpu": [ "x64" ], @@ -3147,6 +2736,29 @@ "optional": true, "os": [ "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.42.tgz", + "integrity": "sha512-N7pQzk9CyE7q0bBN/q0J8s6Db279r5kUZc6d7/wWRe9/zXqC52HQovVyu6iXPIDY4BEzzgbVLhVFXrOuGJ22ZQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.52.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.52.4.tgz", + "integrity": "sha512-Wi6AXf0k0L7E2gteNsNHUs7UMwCIhsCTs6+tqQ5GPwVRWMaflqGec4Sd8n6+FNFDw9vGcReqk2KzBDhCa1DLYg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" ] }, "node_modules/@scalar/openapi-parser": { @@ -3167,9 +2779,9 @@ } }, "node_modules/@smithy/types": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.5.0.tgz", - "integrity": "sha512-RkUpIOsVlAwUIZXO1dsz8Zm+N72LClFfsNqf173catVlvRZiwPy0x2u0JLEA4byreOPKDZPGjmPDylMoP8ZJRg==", + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.7.0.tgz", + "integrity": "sha512-KM8Or+jCDCrUI3wYYhj7ehrC7aATB1NdJ1aFEE/YLKNLVH257k9RNeOqKdg0JOxjyEpVD7KKsmmob9mRy1Ho2g==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" @@ -3201,18 +2813,18 @@ } }, "node_modules/@sveltejs/acorn-typescript": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.5.tgz", - "integrity": "sha512-IwQk4yfwLdibDlrXVE04jTZYlLnwsTT2PIOQQGNLWfjavGifnk1JD1LcZjZaBTRcxZu2FfPfNLOE04DSu9lqtQ==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.6.tgz", + "integrity": "sha512-4awhxtMh4cx9blePWl10HRHj8Iivtqj+2QdDCSMDzxG+XKa9+VCNupQuCuvzEhYPzZSrX+0gC+0lHA/0fFKKQQ==", "license": "MIT", "peerDependencies": { "acorn": "^8.9.0" } }, "node_modules/@sveltejs/adapter-static": { - "version": "3.0.9", - "resolved": "https://registry.npmjs.org/@sveltejs/adapter-static/-/adapter-static-3.0.9.tgz", - "integrity": "sha512-aytHXcMi7lb9ljsWUzXYQ0p5X1z9oWud2olu/EpmH7aCu4m84h7QLvb5Wp+CFirKcwoNnYvYWhyP/L8Vh1ztdw==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@sveltejs/adapter-static/-/adapter-static-3.0.10.tgz", + "integrity": "sha512-7D9lYFWJmB7zxZyTE/qxjksvMqzMuYrrsyh1f4AlZqeZeACPRySjbC3aFiY55wb1tWUaKOQG9PVbm74JcN2Iew==", "dev": true, "license": "MIT", "peerDependencies": { @@ -3220,9 +2832,9 @@ } }, "node_modules/@sveltejs/kit": { - "version": "2.38.0", - "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.38.0.tgz", - "integrity": "sha512-iLmykJOv4PAZvuC0niq1HUoK/LZdfsTW1CpkPAAnroYeYiV7Bp73Eeh/As8u0Y1n/2IDM+p9cdoHYufcpvkXkQ==", + "version": "2.46.5", + "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.46.5.tgz", + "integrity": "sha512-7TSvMrCdmig5TMyYDW876C5FljhA0wlGixtvASCiqUqtLfmyEEpaysXjC7GhR5mWcGRrCGF+L2Bl1eEaW1wTCA==", "dev": true, "license": "MIT", "dependencies": { @@ -3259,9 +2871,9 @@ } }, "node_modules/@sveltejs/package": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@sveltejs/package/-/package-2.5.0.tgz", - "integrity": "sha512-qpB91oWEraOXA4l1ldpGtMc/rLCthbf1ACw/1oroKxvT3sd2NXPd/+NLhIk5FCvd0IUSEZGYa86K+D94GWW2Zw==", + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/@sveltejs/package/-/package-2.5.4.tgz", + "integrity": "sha512-8+1hccAt0M3PPkHVPKH54Wc+cc1PNxRqCrICZiv/hEEto8KwbQVRghxNgTB4htIPyle+4CIB8RayTQH5zRQh9A==", "dev": true, "license": "MIT", "dependencies": { @@ -3282,14 +2894,14 @@ } }, "node_modules/@sveltejs/package/node_modules/svelte2tsx": { - "version": "0.7.42", - "resolved": "https://registry.npmjs.org/svelte2tsx/-/svelte2tsx-0.7.42.tgz", - "integrity": "sha512-PSNrKS16aVdAajoFjpF5M0t6TA7ha7GcKbBajD9RG3M+vooAuvLnWAGUSC6eJL4zEOVbOWKtcS2BuY4rxPljoA==", + "version": "0.7.45", + "resolved": "https://registry.npmjs.org/svelte2tsx/-/svelte2tsx-0.7.45.tgz", + "integrity": "sha512-cSci+mYGygYBHIZLHlm/jYlEc1acjAHqaQaDFHdEBpUueM9kSTnPpvPtSl5VkJOU1qSJ7h1K+6F/LIUYiqC8VA==", "dev": true, "license": "MIT", "dependencies": { "dedent-js": "^1.0.1", - "pascal-case": "^3.1.1" + "scule": "^1.3.0" }, "peerDependencies": { "svelte": "^3.55 || ^4.0.0-next.0 || ^4.0 || ^5.0.0-next.0", @@ -3297,9 +2909,9 @@ } }, "node_modules/@sveltejs/vite-plugin-svelte": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-6.2.0.tgz", - "integrity": "sha512-nJsV36+o7rZUDlrnSduMNl11+RoDE1cKqOI0yUEBCcqFoAZOk47TwD3dPKS2WmRutke9StXnzsPBslY7prDM9w==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-6.2.1.tgz", + "integrity": "sha512-YZs/OSKOQAQCnJvM/P+F1URotNnYNeU3P2s4oIpzm1uFaqUEqRxUB0g5ejMjEb5Gjb9/PiBI5Ktrq4rUUF8UVQ==", "dev": true, "license": "MIT", "dependencies": { @@ -3359,15 +2971,12 @@ } }, "node_modules/@tailwindcss/typography": { - "version": "0.5.16", - "resolved": "https://registry.npmjs.org/@tailwindcss/typography/-/typography-0.5.16.tgz", - "integrity": "sha512-0wDLwCVF5V3x3b1SGXPCDcdsbDHMBe+lkFzBRaHeLvNi+nrrnZ1lA18u+OTWO8iSWU2GxUOCvlXtDuqftc1oiA==", + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/@tailwindcss/typography/-/typography-0.5.19.tgz", + "integrity": "sha512-w31dd8HOx3k9vPtcQh5QHP9GwKcgbMp87j58qi6xgiBnFFtKEAgCWnDw4qUT8aHwkCp8bKvb/KGKWWHedP0AAg==", "dev": true, "license": "MIT", "dependencies": { - "lodash.castarray": "^4.4.0", - "lodash.isplainobject": "^4.0.6", - "lodash.merge": "^4.6.2", "postcss-selector-parser": "6.0.10" }, "peerDependencies": { @@ -3411,9 +3020,9 @@ } }, "node_modules/@tutorlatin/svelte-tiny-virtual-list": { - "version": "3.0.13", - "resolved": "https://registry.npmjs.org/@tutorlatin/svelte-tiny-virtual-list/-/svelte-tiny-virtual-list-3.0.13.tgz", - "integrity": "sha512-MtGad4IQ4qw7r2H15eWa8XkgGnl3egLz5KRdcm4How0ExrY/J82pJxasxsKslJz5nfby81cKOf5E5BmkyRg7gw==", + "version": "3.0.15", + "resolved": "https://registry.npmjs.org/@tutorlatin/svelte-tiny-virtual-list/-/svelte-tiny-virtual-list-3.0.15.tgz", + "integrity": "sha512-ew61aZNXGf0b5X+UjbOAhiNwzI21vijhB/mtBs8bpNOVYQ50TG6Qx00t+fR5C72eGnmdzguewZ2WP6QPNOTQJg==", "license": "MIT", "engines": { "node": ">=20.17.0" @@ -3425,6 +3034,17 @@ "svelte": "^5.0.0" } }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", + "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@types/cookie": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", @@ -3472,9 +3092,9 @@ } }, "node_modules/@types/d3-array": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.1.tgz", - "integrity": "sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.2.tgz", + "integrity": "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw==", "dev": true, "license": "MIT" }, @@ -3791,18 +3411,6 @@ "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", "license": "MIT" }, - "node_modules/@types/node": { - "version": "20.19.13", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.13.tgz", - "integrity": "sha512-yCAeZl7a0DxgNVteXFHt9+uyFbqXGy/ShC4BlcHkoE0AfGXYv/BUiplV72DjMYXHDBXFjhvr6DD1NiRVfB4j8g==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "undici-types": "~6.21.0" - } - }, "node_modules/@types/normalize-package-data": { "version": "2.4.4", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", @@ -3832,9 +3440,9 @@ "license": "MIT" }, "node_modules/@types/vscode": { - "version": "1.103.0", - "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.103.0.tgz", - "integrity": "sha512-o4hanZAQdNfsKecexq9L3eHICd0AAvdbLk6hA60UzGXbGH/q8b/9xv2RgR7vV3ZcHuyKVq7b37IGd/+gM4Tu+Q==", + "version": "1.105.0", + "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.105.0.tgz", + "integrity": "sha512-Lotk3CTFlGZN8ray4VxJE7axIyLZZETQJVWi/lYoUVQuqfRxlQhVOfoejsD2V3dVXPSbS15ov5ZyowMAzgUqcw==", "dev": true, "license": "MIT" }, @@ -4046,17 +3654,6 @@ "integrity": "sha512-bRRFxLfg5dtAyl5XyiVWz/ZBPahpOpPrNYnnHpOpUZvam4tKH35wdhP4Kj6PbM0+KdliOsPzbGWpkxcdpNB/sg==", "license": "MIT" }, - "node_modules/@windmill-labs/esbuild-import-meta-url-plugin": { - "version": "0.0.0-semantic-release", - "resolved": "https://registry.npmjs.org/@windmill-labs/esbuild-import-meta-url-plugin/-/esbuild-import-meta-url-plugin-0.0.0-semantic-release.tgz", - "integrity": "sha512-WaexVcQOpYhX+bRmLq0ZyST1NWu1ujPVLxfcfScP7AX7DUdHfXiLTL1mhdwBcFYiucylOZbkUp2z5lkeSC4abg==", - "dev": true, - "license": "ISC", - "dependencies": { - "esbuild": ">=0.19.x", - "import-meta-resolve": "^4.0.0" - } - }, "node_modules/@windmill-labs/svelte-dnd-action": { "version": "0.9.48", "resolved": "https://registry.npmjs.org/@windmill-labs/svelte-dnd-action/-/svelte-dnd-action-0.9.48.tgz", @@ -4083,22 +3680,22 @@ "peer": true }, "node_modules/@xyflow/svelte": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@xyflow/svelte/-/svelte-1.2.4.tgz", - "integrity": "sha512-CygKmc3t+KevPdx9VEWa6Q0O7DegJ6qzYrOH5dQo5zp9Inm2cYAZpkUuk64ry9Djw/gwy7EvrJTjyXetuvBGOg==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@xyflow/svelte/-/svelte-1.3.1.tgz", + "integrity": "sha512-aLr2v0/nr+zER5+dCzEmR5qCu9l7FCKZwYiRvCX15U2FVIdO2M522pyPEr7Siwq6EEx0QjECACeN+rLZCDSzeA==", "license": "MIT", "dependencies": { "@svelte-put/shortcut": "^4.1.0", - "@xyflow/system": "0.0.68" + "@xyflow/system": "0.0.70" }, "peerDependencies": { "svelte": "^5.25.0" } }, "node_modules/@xyflow/system": { - "version": "0.0.68", - "resolved": "https://registry.npmjs.org/@xyflow/system/-/system-0.0.68.tgz", - "integrity": "sha512-QDG2wxIG4qX+uF8yzm1ULVZrcXX3MxPBoxv7O52FWsX87qIImOqifUhfa/TwsvLdzn7ic2DDBH1uI8TKbdNTYA==", + "version": "0.0.70", + "resolved": "https://registry.npmjs.org/@xyflow/system/-/system-0.0.70.tgz", + "integrity": "sha512-PpC//u9zxdjj0tfTSmZrg3+sRbTz6kop/Amky44U2Dl51sxzDTIUfXMwETOYpmr2dqICWXBIJwXL2a9QWtX2XA==", "license": "MIT", "dependencies": { "@types/d3-drag": "^3.0.7", @@ -4122,6 +3719,18 @@ "svelte": "^3.57.0 || ^4.0.0 || ^5.0.0" } }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/abstract-leveldown": { "version": "6.2.3", "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", @@ -4283,6 +3892,16 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/ansis": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/ansis/-/ansis-4.2.0.tgz", + "integrity": "sha512-HqZ5rWlFjGiV0tDm3UxxgNRqsOTniqoKZu0pIAfh7TZQMGuZK+hH0drySty0si0QXj1ieop4+SkSfPZBPPkHig==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + } + }, "node_modules/any-promise": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", @@ -4412,9 +4031,9 @@ } }, "node_modules/axios": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.11.0.tgz", - "integrity": "sha512-1Lx3WLFQWm3ooKDYZD1eXmoGO9fxYQjrycfHFC8P0sCfQVXyROp0p9PFWBehewBOdCwHc+f/b8I0fMto5eSfwA==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.12.2.tgz", + "integrity": "sha512-vMJzPewAlRyOgxV2dU0Cuz2O8zzzx9VYtbJOaBgXFeLc4IV/Eg50n4LowmehOOR61S8ZMpc2K5Sa7g6A4jfkUw==", "dev": true, "license": "MIT", "dependencies": { @@ -4468,8 +4087,17 @@ "url": "https://feross.org/support" } ], - "license": "MIT", - "optional": true + "license": "MIT" + }, + "node_modules/baseline-browser-mapping": { + "version": "2.8.16", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.16.tgz", + "integrity": "sha512-OMu3BGQ4E7P1ErFsIPpbJh0qvDudM/UuJeHgkAvfWe+0HFJCXh+t/l8L6fVLR55RI/UbKrVLnAXZSVwd9ysWYw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.js" + } }, "node_modules/bezier-easing": { "version": "2.1.0", @@ -4540,10 +4168,16 @@ "node": ">=8" } }, + "node_modules/browser-or-node": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-3.0.0.tgz", + "integrity": "sha512-iczIdVJzGEYhP5DqQxYM9Hh7Ztpqqi+CXZpSmX8ALFs9ecXkQIeqRyM6TfxEfMVpwhl3dSuDvxdzzo9sUOIVBQ==", + "license": "MIT" + }, "node_modules/browserslist": { - "version": "4.25.4", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.25.4.tgz", - "integrity": "sha512-4jYpcjabC606xJ3kw2QwGEZKX0Aw7sgQdZCvIK9dhVSPh76BKo+C+btT1RRofH7B+8iNpEbgGNVWiLki5q93yg==", + "version": "4.26.3", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.26.3.tgz", + "integrity": "sha512-lAUU+02RFBuCKQPj/P6NgjlbCnLBMp4UtgTx7vNHd3XSIJF87s9a5rA3aH2yw3GS9DqZAUbOtZdCCiZeVRqt0w==", "dev": true, "funding": [ { @@ -4561,9 +4195,10 @@ ], "license": "MIT", "dependencies": { - "caniuse-lite": "^1.0.30001737", - "electron-to-chromium": "^1.5.211", - "node-releases": "^2.0.19", + "baseline-browser-mapping": "^2.8.9", + "caniuse-lite": "^1.0.30001746", + "electron-to-chromium": "^1.5.227", + "node-releases": "^2.0.21", "update-browserslist-db": "^1.1.3" }, "bin": { @@ -4826,9 +4461,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001741", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001741.tgz", - "integrity": "sha512-QGUGitqsc8ARjLdgAfxETDhRbJ0REsP6O3I96TAth/mVjh2cYzN2u+3AzPP3aVSm2FehEItaJw1xd+IGBXWeSw==", + "version": "1.0.30001750", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001750.tgz", + "integrity": "sha512-cuom0g5sdX6rw00qOoLNSFCJ9/mYIsuSOA+yzpDw8eopiFqcVwQvZHqov0vmEighRxX++cfC0Vg1G+1Iy/mSpQ==", "dev": true, "funding": [ { @@ -4899,9 +4534,9 @@ } }, "node_modules/chart.js": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.0.tgz", - "integrity": "sha512-aYeC/jDgSEx8SHWZvANYMioYMZ2KX02W6f6uVfyteuCGcadDLcYVHdfdygsTQkQ4TKn5lghoojAsPj5pu0SnvQ==", + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.1.tgz", + "integrity": "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==", "license": "MIT", "dependencies": { "@kurkle/color": "^0.3.0" @@ -4987,6 +4622,12 @@ "node": ">=6" } }, + "node_modules/collection-utils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/collection-utils/-/collection-utils-1.0.1.tgz", + "integrity": "sha512-LA2YTIlR7biSpXkKYwwuzGjwL5rjWEZVOSnvdUc7gObvWe4WkjxOpfrdhoP7Hs09YWDVfg0Mal9BpAqLfVEzQg==", + "license": "Apache-2.0" + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -5109,6 +4750,15 @@ } } }, + "node_modules/cross-fetch": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.1.0.tgz", + "integrity": "sha512-uKm5PU+MHTootlWEY+mZ4vvXoCn4fLQxT9dSc1sXVMSFkINTJVN8cAQROpwcKm8bJ/c7rgZVIBWzH5T78sNZZw==", + "license": "MIT", + "dependencies": { + "node-fetch": "^2.7.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -5125,9 +4775,9 @@ } }, "node_modules/css-declaration-sorter": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/css-declaration-sorter/-/css-declaration-sorter-7.2.0.tgz", - "integrity": "sha512-h70rUM+3PNFuaBDTLe8wF/cdWu+dOZmb7pJt8Z2sedYbAcQVQV/tEchueg3GWxwqS0cxtbxmaHEdkNACqcvsow==", + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/css-declaration-sorter/-/css-declaration-sorter-7.3.0.tgz", + "integrity": "sha512-LQF6N/3vkAMYF4xoHLJfG718HRJh34Z8BnNhd6bosOMIVjMlhuZK5++oZa3uYAgrI5+7x2o27gUqTR2U/KjUOQ==", "dev": true, "license": "ISC", "engines": { @@ -5480,9 +5130,9 @@ "license": "MIT" }, "node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -5716,11 +5366,11 @@ "license": "MIT" }, "node_modules/detect-libc": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz", - "integrity": "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "devOptional": true, "license": "Apache-2.0", - "optional": true, "engines": { "node": ">=8" } @@ -5852,9 +5502,9 @@ } }, "node_modules/dompurify": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.6.tgz", - "integrity": "sha512-/2GogDQlohXPZe6D6NOgQvXLPSYBqIWMnZ8zzOhn09REE4eyAzb+Hed3jhoM9OkuaJ8P6ZGTTVWQKAi8ieIzfQ==", + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.7.tgz", + "integrity": "sha512-WhL/YuveyGXJaerVlMYGWhvQswa7myDG17P7Vu65EWC05o8vfeNbvNf4d/BOvH99+ZW+LlQsc1GDKMa1vNK6dw==", "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" @@ -5933,9 +5583,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.216", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.216.tgz", - "integrity": "sha512-uVgsufJ+qIiOsZBmqkM2AGPn3gbqPySHl/SLKXJ70nowhI0VsRX4aog+R9EUL2bOjqPPhfR9pG8j8s4Zk4xq+A==", + "version": "1.5.235", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.235.tgz", + "integrity": "sha512-i/7ntLFwOdoHY7sgjlTIDo4Sl8EdoTjWIaKinYOVfC6bOp71bmwenyZthWHcasxgHDNWbWxvG9M3Ia116zIaYQ==", "dev": true, "license": "ISC" }, @@ -5998,9 +5648,9 @@ } }, "node_modules/error-ex": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", - "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", + "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", "dev": true, "license": "MIT", "peer": true, @@ -6054,48 +5704,6 @@ "node": ">= 0.4" } }, - "node_modules/esbuild": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.9.tgz", - "integrity": "sha512-CRbODhYyQx3qp7ZEwzxOk4JBqmD/seJrzPa/cGjY1VtIn5E09Oi9/dB4JwctnfZ8Q8iT7rioVv5k/FNT/uf54g==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.9", - "@esbuild/android-arm": "0.25.9", - "@esbuild/android-arm64": "0.25.9", - "@esbuild/android-x64": "0.25.9", - "@esbuild/darwin-arm64": "0.25.9", - "@esbuild/darwin-x64": "0.25.9", - "@esbuild/freebsd-arm64": "0.25.9", - "@esbuild/freebsd-x64": "0.25.9", - "@esbuild/linux-arm": "0.25.9", - "@esbuild/linux-arm64": "0.25.9", - "@esbuild/linux-ia32": "0.25.9", - "@esbuild/linux-loong64": "0.25.9", - "@esbuild/linux-mips64el": "0.25.9", - "@esbuild/linux-ppc64": "0.25.9", - "@esbuild/linux-riscv64": "0.25.9", - "@esbuild/linux-s390x": "0.25.9", - "@esbuild/linux-x64": "0.25.9", - "@esbuild/netbsd-arm64": "0.25.9", - "@esbuild/netbsd-x64": "0.25.9", - "@esbuild/openbsd-arm64": "0.25.9", - "@esbuild/openbsd-x64": "0.25.9", - "@esbuild/openharmony-arm64": "0.25.9", - "@esbuild/sunos-x64": "0.25.9", - "@esbuild/win32-arm64": "0.25.9", - "@esbuild/win32-ia32": "0.25.9", - "@esbuild/win32-x64": "0.25.9" - } - }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -6514,12 +6122,30 @@ "node": ">=0.10.0" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/eventemitter3": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-2.0.3.tgz", "integrity": "sha512-jLN68Dx5kyFHaePoXWPsCGW5qdyZQtLYHkxkg02/Mz6g0kYpDx4FyP6XfArhQdlOC4b8Mv+EMxPo/8La7Tzghg==", "license": "MIT" }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/expand-template": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", @@ -6549,9 +6175,9 @@ "license": "Apache-2.0" }, "node_modules/fast-equals": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-5.2.2.tgz", - "integrity": "sha512-V7/RktU11J3I36Nwq2JnZEM7tNm17eBJz+u25qdxBZeCKiX6BkVSZQjwWIr+IobgnZy+ag73tTZgZi7tr0LrBw==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-5.3.2.tgz", + "integrity": "sha512-6rxyATwPCkaFIL3JLqw8qXqMpIZ942pTX/tbQFkRsDGblS8tNGtlUauA/+mt6RUfqn/4MoEr+WDkYoIQbibWuQ==", "license": "MIT", "engines": { "node": ">=6.0.0" @@ -6990,19 +6616,6 @@ "node": ">=8" } }, - "node_modules/giget/node_modules/mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "dev": true, - "license": "MIT", - "bin": { - "mkdirp": "bin/cmd.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/giget/node_modules/pathe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", @@ -7608,17 +7221,6 @@ "node": ">=8" } }, - "node_modules/import-meta-resolve": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.2.0.tgz", - "integrity": "sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg==", - "dev": true, - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -7856,6 +7458,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-url": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", + "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==", + "license": "MIT" + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -7906,6 +7514,12 @@ "jiti": "bin/jiti.js" } }, + "node_modules/js-base64": { + "version": "3.7.8", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.8.tgz", + "integrity": "sha512-hNngCeKxIUQiEUN3GPJOkz4wF/YvdUdbNL9hsBcMQTkKzboD7T/q3OYOuuPZLUE6dBxSGpwhk5mwuDud7JVAow==", + "license": "BSD-3-Clause" + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -8174,9 +7788,9 @@ } }, "node_modules/leven": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-4.0.0.tgz", - "integrity": "sha512-puehA3YKku3osqPlNuzGDUHq8WpwXupUg1V6NXdV38G+gr+gkBwFC8g1b/+YcIvp8gnqVIus+eJCH/eGsRmJNw==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-4.1.0.tgz", + "integrity": "sha512-KZ9W9nWDT7rF7Dazg8xyLHGLrmpgq2nVNFUckhqdW3szVP6YhCpp/RAnpmVExA9JvrMynjwSLVrEj3AepHR6ew==", "license": "MIT", "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" @@ -8220,6 +7834,267 @@ "url": "https://github.com/sponsors/dmonad" } }, + "node_modules/lightningcss": { + "version": "1.30.2", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.30.2.tgz", + "integrity": "sha512-utfs7Pr5uJyyvDETitgsaqSyjCb2qNRAtuqUeWIAKztsOYdcACf2KtARYXg2pSvhkt+9NfoaNY7fxjl6nuMjIQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.30.2", + "lightningcss-darwin-arm64": "1.30.2", + "lightningcss-darwin-x64": "1.30.2", + "lightningcss-freebsd-x64": "1.30.2", + "lightningcss-linux-arm-gnueabihf": "1.30.2", + "lightningcss-linux-arm64-gnu": "1.30.2", + "lightningcss-linux-arm64-musl": "1.30.2", + "lightningcss-linux-x64-gnu": "1.30.2", + "lightningcss-linux-x64-musl": "1.30.2", + "lightningcss-win32-arm64-msvc": "1.30.2", + "lightningcss-win32-x64-msvc": "1.30.2" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.30.2", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.30.2.tgz", + "integrity": "sha512-BH9sEdOCahSgmkVhBLeU7Hc9DWeZ1Eb6wNS6Da8igvUwAe0sqROHddIlvU06q3WyXVEOYDZ6ykBZQnjTbmo4+A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.30.2", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.30.2.tgz", + "integrity": "sha512-ylTcDJBN3Hp21TdhRT5zBOIi73P6/W0qwvlFEk22fkdXchtNTOU4Qc37SkzV+EKYxLouZ6M4LG9NfZ1qkhhBWA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.30.2", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.30.2.tgz", + "integrity": "sha512-oBZgKchomuDYxr7ilwLcyms6BCyLn0z8J0+ZZmfpjwg9fRVZIR5/GMXd7r9RH94iDhld3UmSjBM6nXWM2TfZTQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.30.2", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.30.2.tgz", + "integrity": "sha512-c2bH6xTrf4BDpK8MoGG4Bd6zAMZDAXS569UxCAGcA7IKbHNMlhGQ89eRmvpIUGfKWNVdbhSbkQaWhEoMGmGslA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.30.2", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.30.2.tgz", + "integrity": "sha512-eVdpxh4wYcm0PofJIZVuYuLiqBIakQ9uFZmipf6LF/HRj5Bgm0eb3qL/mr1smyXIS1twwOxNWndd8z0E374hiA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.30.2", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.30.2.tgz", + "integrity": "sha512-UK65WJAbwIJbiBFXpxrbTNArtfuznvxAJw4Q2ZGlU8kPeDIWEX1dg3rn2veBVUylA2Ezg89ktszWbaQnxD/e3A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.30.2", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.30.2.tgz", + "integrity": "sha512-5Vh9dGeblpTxWHpOx8iauV02popZDsCYMPIgiuw97OJ5uaDsL86cnqSFs5LZkG3ghHoX5isLgWzMs+eD1YzrnA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.30.2", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.30.2.tgz", + "integrity": "sha512-Cfd46gdmj1vQ+lR6VRTTadNHu6ALuw2pKR9lYq4FnhvgBc4zWY1EtZcAc6EffShbb1MFrIPfLDXD6Xprbnni4w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.30.2", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.30.2.tgz", + "integrity": "sha512-XJaLUUFXb6/QG2lGIW6aIk6jKdtjtcffUT0NKvIqhSBY3hh9Ch+1LCeH80dR9q9LBjG3ewbDjnumefsLsP6aiA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.30.2", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.30.2.tgz", + "integrity": "sha512-FZn+vaj7zLv//D/192WFFVA0RgHawIcHqLX9xuWiQt7P0PtdFEVaxgF9rjM/IRYHQXNnk61/H/gb2Ei+kUQ4xQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.30.2", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.30.2.tgz", + "integrity": "sha512-5g1yc73p+iAkid5phb4oVFMB45417DkRevRbt/El/gKXJk4jid+vPFF/AXbxn05Aky8PapwzZrdJShv5C0avjw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, "node_modules/lilconfig": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", @@ -8262,11 +8137,10 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/lodash.castarray": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.castarray/-/lodash.castarray-4.4.0.tgz", - "integrity": "sha512-aVx8ztPv7/2ULbArGJ2Y42bG1mEQ5mGjpdvrbJcJFU3TbYybe+QlLS4pst9zV52ymy2in1KpFPiZnAOATxD4+Q==", - "dev": true, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "license": "MIT" }, "node_modules/lodash.clonedeep": { @@ -8295,13 +8169,6 @@ "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", "license": "MIT" }, - "node_modules/lodash.isplainobject": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", - "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", - "dev": true, - "license": "MIT" - }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -8352,9 +8219,9 @@ } }, "node_modules/lru-cache": { - "version": "11.2.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.1.tgz", - "integrity": "sha512-r8LA6i4LP4EeWOhqBaZZjDWwehd1xUJPCJd9Sv300H0ZmcUER4+JPh7bqqZeqs1o5pgtgvXm+d9UGrB5zZGDiQ==", + "version": "11.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.2.tgz", + "integrity": "sha512-F9ODfyqML2coTIsQpSkRHnLSZMtkU8Q+mSfcaIyKwy58u+8k5nvAYeiNhsyMARvzNcXJ9QfWVrcPsC9e9rAxtg==", "license": "ISC", "engines": { "node": "20 || >=22" @@ -9342,17 +9209,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/min-indent": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", - "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", - "dev": true, - "license": "MIT", - "peer": true, - "engines": { - "node": ">=4" - } - }, "node_modules/mini-svg-data-uri": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/mini-svg-data-uri/-/mini-svg-data-uri-1.4.4.tgz", @@ -9414,9 +9270,9 @@ } }, "node_modules/minizlib": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.2.tgz", - "integrity": "sha512-oG62iEk+CYt5Xj2YqI5Xi9xWUeZhDI8jjQmC5oThVH5JGCTgIjr7ciJDzC7MBzYd//WvR1OTmP5Q38Q8ShQtVA==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", "dev": true, "license": "MIT", "dependencies": { @@ -9427,19 +9283,16 @@ } }, "node_modules/mkdirp": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", - "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", "dev": true, "license": "MIT", "bin": { - "mkdirp": "dist/cjs/src/bin.js" + "mkdirp": "bin/cmd.js" }, "engines": { "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" } }, "node_modules/mkdirp-classic": { @@ -9471,54 +9324,13 @@ }, "node_modules/monaco-editor": { "name": "@codingame/monaco-vscode-editor-api", - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-editor-api/-/monaco-vscode-editor-api-20.2.1.tgz", - "integrity": "sha512-f+e6Lchp/aW2J5lEqkULP8NF4PGRayVG2/X90HN7ydlWAIr/WDZfVDmP2XhmnTZXfAhCoYQWWljtHMQWY7aSyw==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-editor-api/-/monaco-vscode-editor-api-21.6.0.tgz", + "integrity": "sha512-YTxKRHe9d4TvyEzWIqLpJXLyZyO4xFlLgrkgHoWBpomm6gIuwaRJJRpapBZf24oG8AhkniZSPg2iv/84M+ho6g==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-5452e2b7-9081-5f95-839b-4ab3544ce28f-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1" - } - }, - "node_modules/monaco-editor-wrapper": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/monaco-editor-wrapper/-/monaco-editor-wrapper-6.12.0.tgz", - "integrity": "sha512-4rn5pvcBUPK5OjQikHCCsda5DrtswCWJ5+iU27uDTTV4uTSn2BY0Rp73aNXWIF5VsgNJ7/sEf/5DpTIQP8o/HA==", - "license": "MIT", - "dependencies": { - "@codingame/monaco-vscode-api": "~20.2.1", - "@codingame/monaco-vscode-editor-api": "~20.2.1", - "@codingame/monaco-vscode-editor-service-override": "~20.2.1", - "@codingame/monaco-vscode-extension-api": "~20.2.1", - "@codingame/monaco-vscode-language-pack-cs": "~20.2.1", - "@codingame/monaco-vscode-language-pack-de": "~20.2.1", - "@codingame/monaco-vscode-language-pack-es": "~20.2.1", - "@codingame/monaco-vscode-language-pack-fr": "~20.2.1", - "@codingame/monaco-vscode-language-pack-it": "~20.2.1", - "@codingame/monaco-vscode-language-pack-ja": "~20.2.1", - "@codingame/monaco-vscode-language-pack-ko": "~20.2.1", - "@codingame/monaco-vscode-language-pack-pl": "~20.2.1", - "@codingame/monaco-vscode-language-pack-pt-br": "~20.2.1", - "@codingame/monaco-vscode-language-pack-qps-ploc": "~20.2.1", - "@codingame/monaco-vscode-language-pack-ru": "~20.2.1", - "@codingame/monaco-vscode-language-pack-tr": "~20.2.1", - "@codingame/monaco-vscode-language-pack-zh-hans": "~20.2.1", - "@codingame/monaco-vscode-language-pack-zh-hant": "~20.2.1", - "@codingame/monaco-vscode-monarch-service-override": "~20.2.1", - "@codingame/monaco-vscode-textmate-service-override": "~20.2.1", - "@codingame/monaco-vscode-theme-defaults-default-extension": "~20.2.1", - "@codingame/monaco-vscode-theme-service-override": "~20.2.1", - "@codingame/monaco-vscode-views-service-override": "~20.2.1", - "@codingame/monaco-vscode-workbench-service-override": "~20.2.1", - "monaco-languageclient": "~9.11.0", - "vscode": "npm:@codingame/monaco-vscode-extension-api@~20.2.1", - "vscode-languageclient": "~9.0.1", - "vscode-languageserver-protocol": "~3.17.5", - "vscode-ws-jsonrpc": "~3.5.0" - }, - "engines": { - "node": ">=20.10.0", - "npm": ">=10.2.3" + "@codingame/monaco-vscode-5452e2b7-9081-5f95-839b-4ab3544ce28f-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0" } }, "node_modules/monaco-graphql": { @@ -9537,23 +9349,44 @@ } }, "node_modules/monaco-languageclient": { - "version": "9.11.0", - "resolved": "https://registry.npmjs.org/monaco-languageclient/-/monaco-languageclient-9.11.0.tgz", - "integrity": "sha512-76aEPzISqQF/6W6eAonWWcAiAqYsNlo7CeKxhAgXokGEw51mS4SraEumwhyHeb0uM6KWkTpzEdxShXxBAmbtGw==", + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/monaco-languageclient/-/monaco-languageclient-10.1.0.tgz", + "integrity": "sha512-aWHi84S9MKnkppZoNOCL7bRU3gY1xbkjExMOjp8EQtKe0o+gsQcmigCymzIc1C5XFCU395ENG5AKWeseRXjQsQ==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-api": "~20.2.1", - "@codingame/monaco-vscode-configuration-service-override": "~20.2.1", - "@codingame/monaco-vscode-editor-api": "~20.2.1", - "@codingame/monaco-vscode-editor-service-override": "~20.2.1", - "@codingame/monaco-vscode-extension-api": "~20.2.1", - "@codingame/monaco-vscode-extensions-service-override": "~20.2.1", - "@codingame/monaco-vscode-languages-service-override": "~20.2.1", - "@codingame/monaco-vscode-localization-service-override": "~20.2.1", - "@codingame/monaco-vscode-log-service-override": "~20.2.1", - "@codingame/monaco-vscode-model-service-override": "~20.2.1", - "vscode": "npm:@codingame/monaco-vscode-extension-api@~20.2.1", - "vscode-languageclient": "~9.0.1" + "@codingame/monaco-vscode-api": "^21.3.2", + "@codingame/monaco-vscode-configuration-service-override": "^21.3.2", + "@codingame/monaco-vscode-editor-api": "^21.3.2", + "@codingame/monaco-vscode-editor-service-override": "^21.3.2", + "@codingame/monaco-vscode-extension-api": "^21.3.2", + "@codingame/monaco-vscode-extensions-service-override": "^21.3.2", + "@codingame/monaco-vscode-language-pack-cs": "^21.3.2", + "@codingame/monaco-vscode-language-pack-de": "^21.3.2", + "@codingame/monaco-vscode-language-pack-es": "^21.3.2", + "@codingame/monaco-vscode-language-pack-fr": "^21.3.2", + "@codingame/monaco-vscode-language-pack-it": "^21.3.2", + "@codingame/monaco-vscode-language-pack-ja": "^21.3.2", + "@codingame/monaco-vscode-language-pack-ko": "^21.3.2", + "@codingame/monaco-vscode-language-pack-pl": "^21.3.2", + "@codingame/monaco-vscode-language-pack-pt-br": "^21.3.2", + "@codingame/monaco-vscode-language-pack-qps-ploc": "^21.3.2", + "@codingame/monaco-vscode-language-pack-ru": "^21.3.2", + "@codingame/monaco-vscode-language-pack-tr": "^21.3.2", + "@codingame/monaco-vscode-language-pack-zh-hans": "^21.3.2", + "@codingame/monaco-vscode-language-pack-zh-hant": "^21.3.2", + "@codingame/monaco-vscode-languages-service-override": "^21.3.2", + "@codingame/monaco-vscode-localization-service-override": "^21.3.2", + "@codingame/monaco-vscode-log-service-override": "^21.3.2", + "@codingame/monaco-vscode-model-service-override": "^21.3.2", + "@codingame/monaco-vscode-monarch-service-override": "^21.3.2", + "@codingame/monaco-vscode-textmate-service-override": "^21.3.2", + "@codingame/monaco-vscode-theme-defaults-default-extension": "^21.3.2", + "@codingame/monaco-vscode-theme-service-override": "^21.3.2", + "@codingame/monaco-vscode-views-service-override": "^21.3.2", + "@codingame/monaco-vscode-workbench-service-override": "^21.3.2", + "vscode-languageclient": "~9.0.1", + "vscode-languageserver-protocol": "~3.17.5", + "vscode-ws-jsonrpc": "~3.5.0" }, "engines": { "node": ">=20.10.0", @@ -9608,9 +9441,9 @@ } }, "node_modules/nanoid": { - "version": "5.1.5", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.5.tgz", - "integrity": "sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw==", + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.6.tgz", + "integrity": "sha512-c7+7RQ+dMB5dPwwCp4ee1/iV/q2P6aK1mTZcfr1BTuVlyW9hJYiMPybJCcnBlQtuSmTIWNeazm/zqNoZSSElBg==", "dev": true, "funding": [ { @@ -9662,9 +9495,9 @@ "license": "MIT" }, "node_modules/ngraph.events": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/ngraph.events/-/ngraph.events-1.3.2.tgz", - "integrity": "sha512-38oe7gGmJec5V6Ejo5L3EWOpy2coJ5ZjY9oo9Sz1bE4wqoVorn/5iejGHtdR+5EwO6d1qOHqgqZxHt9dLqiLTw==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/ngraph.events/-/ngraph.events-1.4.0.tgz", + "integrity": "sha512-NeDGI4DSyjBNBRtA86222JoYietsmCXbs8CEB0dZ51Xeh4lhVl1y3wpWLumczvnha8sFQIW4E0vvVWwgmX2mGw==", "license": "BSD-3-Clause" }, "node_modules/no-case": { @@ -9679,9 +9512,9 @@ } }, "node_modules/node-abi": { - "version": "3.77.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.77.0.tgz", - "integrity": "sha512-DSmt0OEcLoK4i3NuscSbGjOf3bqiDEutejqENSplMSFA/gmB8mkED9G4pKWnPl7MDU4rSHebKPHeitpDfyH0cQ==", + "version": "3.78.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.78.0.tgz", + "integrity": "sha512-E2wEyrgX/CqvicaQYU3Ze1PFGjc4QYPGsjUrlYkqAE0WjHEZwgOsGMPMzkMse4LjJbDmaEuDX3CM036j5K2DSQ==", "license": "MIT", "optional": true, "dependencies": { @@ -9698,6 +9531,26 @@ "license": "MIT", "optional": true }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-fetch-native": { "version": "1.6.7", "resolved": "https://registry.npmjs.org/node-fetch-native/-/node-fetch-native-1.6.7.tgz", @@ -9718,9 +9571,9 @@ } }, "node_modules/node-releases": { - "version": "2.0.20", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.20.tgz", - "integrity": "sha512-7gK6zSXEH6neM212JgfYFXe+GmZQM+fia5SsusuBIUgnPheLFBmIPhtFoAQRj8/7wASYQnbDlHPVwY0BefoFgA==", + "version": "2.0.23", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.23.tgz", + "integrity": "sha512-cCmFDMSm26S6tQSDpBCg/NR8NENrVPhAJSf+XbxBG4rPFaaonlEoE9wHQmun+cls499TQGSb7ZyPBRlzgKfpeg==", "dev": true, "license": "MIT" }, @@ -9899,9 +9752,9 @@ } }, "node_modules/openai": { - "version": "5.20.1", - "resolved": "https://registry.npmjs.org/openai/-/openai-5.20.1.tgz", - "integrity": "sha512-UndCB0R5V3iB9I98NyF69zNP6YfwU4+Fjk0eW4HhooTm+Awlpm/MGjJTwJsyNV/qkH1NJi0GG+9odwukGTqExQ==", + "version": "5.23.2", + "resolved": "https://registry.npmjs.org/openai/-/openai-5.23.2.tgz", + "integrity": "sha512-MQBzmTulj+MM5O8SKEk/gL8a7s5mktS9zUtAkU257WjvobGc9nKcBuVwjyEEcb9SI8a8Y2G/mzn3vm9n1Jlleg==", "license": "Apache-2.0", "bin": { "openai": "bin/cli" @@ -10295,13 +10148,13 @@ "license": "MIT" }, "node_modules/playwright": { - "version": "1.55.0", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.55.0.tgz", - "integrity": "sha512-sdCWStblvV1YU909Xqx0DhOjPZE4/5lJsIS84IfN9dAZfcl/CIZ5O8l3o0j7hPMjDvqoTF8ZUcc+i/GL5erstA==", + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.56.0.tgz", + "integrity": "sha512-X5Q1b8lOdWIE4KAoHpW3SE8HvUB+ZZsUoN64ZhjnN8dOb1UpujxBtENGiZFE+9F/yhzJwYa+ca3u43FeLbboHA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.55.0" + "playwright-core": "1.56.0" }, "bin": { "playwright": "cli.js" @@ -10314,9 +10167,9 @@ } }, "node_modules/playwright-core": { - "version": "1.55.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.55.0.tgz", - "integrity": "sha512-GvZs4vU3U5ro2nZpeiwyb0zuFaqb9sUiAJuyrWpcGouD8y9/HLgGbNRjIph7zU9D3hnPaisMl9zG9CgFi/biIg==", + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.56.0.tgz", + "integrity": "sha512-1SXl7pMfemAMSDn5rkPeZljxOCYAmQnYLBTExuh6E8USHXGSX3dx6lYZN/xPpTz1vimXmPA9CDnILvmJaB8aSQ==", "dev": true, "license": "Apache-2.0", "bin": { @@ -10341,6 +10194,15 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/pluralize": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", + "integrity": "sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/postcss": { "version": "8.5.6", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", @@ -10508,10 +10370,20 @@ } }, "node_modules/postcss-js": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/postcss-js/-/postcss-js-4.0.1.tgz", - "integrity": "sha512-dDLF8pEO191hJMtlHFPRa8xsizHaM82MLfNkUHdUtVEV3tgTp5oj+8qbEqYM57SLfc74KSbw//4SeJma2LRVIw==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/postcss-js/-/postcss-js-4.1.0.tgz", + "integrity": "sha512-oIAOTqgIo7q2EOwbhb8UalYePMvYoIeRY2YKntdpFQXNosSu3vLrniGgmH9OKs/qAkfoj5oB3le/7mINW1LCfw==", "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], "license": "MIT", "dependencies": { "camelcase-css": "^2.0.1" @@ -10519,10 +10391,6 @@ "engines": { "node": "^12 || ^14 || >= 16" }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, "peerDependencies": { "postcss": "^8.4.21" } @@ -11129,6 +10997,15 @@ "svelte": "^3.2.0 || ^4.0.0-next.0 || ^5.0.0-next.0" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/property-information": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.1.0.tgz", @@ -11229,6 +11106,74 @@ "integrity": "sha512-RKJ22hX8mHe3Y6wH/N3wCM6BWtjaxIyyUIkpHOvfFnxdI4yD4tBXEBKSbriGujF6jnSVkJrffuo6vxACiSSxIw==", "license": "ISC" }, + "node_modules/quicktype-core": { + "version": "23.2.6", + "resolved": "https://registry.npmjs.org/quicktype-core/-/quicktype-core-23.2.6.tgz", + "integrity": "sha512-asfeSv7BKBNVb9WiYhFRBvBZHcRutPRBwJMxW0pefluK4kkKu4lv0IvZBwFKvw2XygLcL1Rl90zxWDHYgkwCmA==", + "license": "Apache-2.0", + "dependencies": { + "@glideapps/ts-necessities": "2.2.3", + "browser-or-node": "^3.0.0", + "collection-utils": "^1.0.1", + "cross-fetch": "^4.0.0", + "is-url": "^1.2.4", + "js-base64": "^3.7.7", + "lodash": "^4.17.21", + "pako": "^1.0.6", + "pluralize": "^8.0.0", + "readable-stream": "4.5.2", + "unicode-properties": "^1.4.1", + "urijs": "^1.19.1", + "wordwrap": "^1.0.0", + "yaml": "^2.4.1" + } + }, + "node_modules/quicktype-core/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/quicktype-core/node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, + "node_modules/quicktype-core/node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/quill": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/quill/-/quill-1.3.7.tgz", @@ -11622,45 +11567,38 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/rollup": { - "version": "4.50.1", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.50.1.tgz", - "integrity": "sha512-78E9voJHwnXQMiQdiqswVLZwJIzdBKJ1GdI5Zx6XwoFKUIk09/sSrr+05QFzvYb8q6Y9pPV45zzDuYa3907TZA==", + "node_modules/rolldown": { + "version": "1.0.0-beta.42", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-beta.42.tgz", + "integrity": "sha512-xaPcckj+BbJhYLsv8gOqezc8EdMcKKe/gk8v47B0KPvgABDrQ0qmNPAiT/gh9n9Foe0bUkEv2qzj42uU5q1WRg==", "dev": true, "license": "MIT", "dependencies": { - "@types/estree": "1.0.8" + "@oxc-project/types": "=0.94.0", + "@rolldown/pluginutils": "1.0.0-beta.42", + "ansis": "=4.2.0" }, "bin": { - "rollup": "dist/bin/rollup" + "rolldown": "bin/cli.mjs" }, "engines": { - "node": ">=18.0.0", - "npm": ">=8.0.0" + "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.50.1", - "@rollup/rollup-android-arm64": "4.50.1", - "@rollup/rollup-darwin-arm64": "4.50.1", - "@rollup/rollup-darwin-x64": "4.50.1", - "@rollup/rollup-freebsd-arm64": "4.50.1", - "@rollup/rollup-freebsd-x64": "4.50.1", - "@rollup/rollup-linux-arm-gnueabihf": "4.50.1", - "@rollup/rollup-linux-arm-musleabihf": "4.50.1", - "@rollup/rollup-linux-arm64-gnu": "4.50.1", - "@rollup/rollup-linux-arm64-musl": "4.50.1", - "@rollup/rollup-linux-loongarch64-gnu": "4.50.1", - "@rollup/rollup-linux-ppc64-gnu": "4.50.1", - "@rollup/rollup-linux-riscv64-gnu": "4.50.1", - "@rollup/rollup-linux-riscv64-musl": "4.50.1", - "@rollup/rollup-linux-s390x-gnu": "4.50.1", - "@rollup/rollup-linux-x64-gnu": "4.50.1", - "@rollup/rollup-linux-x64-musl": "4.50.1", - "@rollup/rollup-openharmony-arm64": "4.50.1", - "@rollup/rollup-win32-arm64-msvc": "4.50.1", - "@rollup/rollup-win32-ia32-msvc": "4.50.1", - "@rollup/rollup-win32-x64-msvc": "4.50.1", - "fsevents": "~2.3.2" + "@rolldown/binding-android-arm64": "1.0.0-beta.42", + "@rolldown/binding-darwin-arm64": "1.0.0-beta.42", + "@rolldown/binding-darwin-x64": "1.0.0-beta.42", + "@rolldown/binding-freebsd-x64": "1.0.0-beta.42", + "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-beta.42", + "@rolldown/binding-linux-arm64-gnu": "1.0.0-beta.42", + "@rolldown/binding-linux-arm64-musl": "1.0.0-beta.42", + "@rolldown/binding-linux-x64-gnu": "1.0.0-beta.42", + "@rolldown/binding-linux-x64-musl": "1.0.0-beta.42", + "@rolldown/binding-openharmony-arm64": "1.0.0-beta.42", + "@rolldown/binding-wasm32-wasi": "1.0.0-beta.42", + "@rolldown/binding-win32-arm64-msvc": "1.0.0-beta.42", + "@rolldown/binding-win32-ia32-msvc": "1.0.0-beta.42", + "@rolldown/binding-win32-x64-msvc": "1.0.0-beta.42" } }, "node_modules/run-parallel": { @@ -11724,13 +11662,19 @@ "url": "https://feross.org/support" } ], - "license": "MIT", - "optional": true + "license": "MIT" + }, + "node_modules/scule": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/scule/-/scule-1.3.0.tgz", + "integrity": "sha512-6FtHJEvt+pVMIB9IBY+IcCJ6Z5f1iQnytgyfKMhDKgmzYG+TeH/wx1y3l27rshSbLiSanrR9ffZDrEsmjlQF2g==", + "dev": true, + "license": "MIT" }, "node_modules/semver": { - "version": "7.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", - "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -12008,7 +11952,6 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", "license": "MIT", - "optional": true, "dependencies": { "safe-buffer": "~5.2.0" } @@ -12070,15 +12013,12 @@ } }, "node_modules/strip-indent": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-4.0.0.tgz", - "integrity": "sha512-mnVSV2l+Zv6BLpSD/8V87CW/y9EmmbYzGCIavsnsI6/nwn26DwffM/yztm30Z/I2DY9wdS3vXVCMnHDgZaVNoA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-4.1.1.tgz", + "integrity": "sha512-SlyRoSkdh1dYP0PzclLE7r0M9sgbFKKMFXpFRUMNuKhQSbC6VQIGzq3E0qsfvGJaUFJPGv6Ws1NZ/haTAjfbMA==", "dev": true, "license": "MIT", "peer": true, - "dependencies": { - "min-indent": "^1.0.1" - }, "engines": { "node": ">=12" }, @@ -12425,9 +12365,9 @@ } }, "node_modules/svelte": { - "version": "5.38.8", - "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.38.8.tgz", - "integrity": "sha512-UDpTbM/iuZ4MaMnn4ODB3rf5JKDyPOi5oJcopP0j7YHQ9BuJtsAqsR71r2N6AnJf7ygbalTJU5y8eSWGAQZjlQ==", + "version": "5.39.12", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.39.12.tgz", + "integrity": "sha512-CEzwxFuEycokU8K8CE/OuwVbmei+ivu2HvBGYIdASfMa1hCRSNr4RRkzNSvbAvu6h+BOig2CsZTAEY+WKvwZpA==", "license": "MIT", "dependencies": { "@jridgewell/remapping": "^2.3.4", @@ -12483,9 +12423,9 @@ } }, "node_modules/svelte-check": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/svelte-check/-/svelte-check-4.3.1.tgz", - "integrity": "sha512-lkh8gff5gpHLjxIV+IaApMxQhTGnir2pNUAqcNgeKkvK5bT/30Ey/nzBxNLDlkztCH4dP7PixkMt9SWEKFPBWg==", + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/svelte-check/-/svelte-check-4.3.3.tgz", + "integrity": "sha512-RYP0bEwenDXzfv0P1sKAwjZSlaRyqBn0Fz1TVni58lqyEiqgwztTpmodJrGzP6ZT2aHl4MbTvWP6gbmQ3FOnBg==", "dev": true, "license": "MIT", "dependencies": { @@ -12816,9 +12756,9 @@ } }, "node_modules/tailwindcss": { - "version": "3.4.17", - "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.17.tgz", - "integrity": "sha512-w33E2aCvSDP0tW9RZuNXadXlkHXqFzSkQew/aIa2i/Sj8fThxwovwlXHSPXTbAHwEIhBFXAedUhP2tueAKP8Og==", + "version": "3.4.18", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.18.tgz", + "integrity": "sha512-6A2rnmW5xZMdw11LYjhcI5846rt9pbLSabY5XPxo+XWdxwZaFEn47Go4NzFiHu9sNNmr/kXivP1vStfvMaK1GQ==", "dev": true, "license": "MIT", "dependencies": { @@ -12830,7 +12770,7 @@ "fast-glob": "^3.3.2", "glob-parent": "^6.0.2", "is-glob": "^4.0.3", - "jiti": "^1.21.6", + "jiti": "^1.21.7", "lilconfig": "^3.1.3", "micromatch": "^4.0.8", "normalize-path": "^3.0.0", @@ -12839,7 +12779,7 @@ "postcss": "^8.4.47", "postcss-import": "^15.1.0", "postcss-js": "^4.0.1", - "postcss-load-config": "^4.0.2", + "postcss-load-config": "^4.0.2 || ^5.0 || ^6.0", "postcss-nested": "^6.2.0", "postcss-selector-parser": "^6.1.2", "resolve": "^1.22.8", @@ -12919,17 +12859,16 @@ } }, "node_modules/tar": { - "version": "7.4.3", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", - "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", + "version": "7.5.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.1.tgz", + "integrity": "sha512-nlGpxf+hv0v7GkWBK2V9spgactGOp0qvfWRxUMjqHyzrt3SgwE48DIv/FhqPHJYLHpgW1opq3nERbz5Anq7n1g==", "dev": true, "license": "ISC", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", - "minizlib": "^3.0.1", - "mkdirp": "^3.0.1", + "minizlib": "^3.1.0", "yallist": "^5.0.0" }, "engines": { @@ -12937,9 +12876,9 @@ } }, "node_modules/tar-fs": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.3.tgz", - "integrity": "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", "license": "MIT", "optional": true, "dependencies": { @@ -13003,6 +12942,12 @@ "node": ">=0.8" } }, + "node_modules/tiny-inflate": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tiny-inflate/-/tiny-inflate-1.0.3.tgz", + "integrity": "sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw==", + "license": "MIT" + }, "node_modules/tinyexec": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", @@ -13081,6 +13026,12 @@ "node": ">=6" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/trim-lines": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", @@ -13191,9 +13142,9 @@ } }, "node_modules/typescript": { - "version": "5.9.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.2.tgz", - "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", "bin": { @@ -13225,14 +13176,31 @@ "node": ">=0.8.0" } }, - "node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, + "node_modules/unicode-properties": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/unicode-properties/-/unicode-properties-1.4.1.tgz", + "integrity": "sha512-CLjCCLQ6UuMxWnbIylkisbRj31qxHPAurvena/0iwSVbQ2G1VY5/HjV0IRabOEbDHlzZlRdCrD4NhB0JtU40Pg==", "license": "MIT", - "optional": true, - "peer": true + "dependencies": { + "base64-js": "^1.3.0", + "unicode-trie": "^2.0.0" + } + }, + "node_modules/unicode-trie": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-trie/-/unicode-trie-2.0.0.tgz", + "integrity": "sha512-x7bc76x0bm4prf1VLg79uhAzKw8DVboClSN5VxJuQ+LKDOVEW9CdH+VY7SP+vX7xCYQqzzgQpFqz15zeLvAtZQ==", + "license": "MIT", + "dependencies": { + "pako": "^0.2.5", + "tiny-inflate": "^1.0.0" + } + }, + "node_modules/unicode-trie/node_modules/pako": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", + "integrity": "sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==", + "license": "MIT" }, "node_modules/unified": { "version": "11.0.5", @@ -13374,6 +13342,12 @@ "punycode": "^2.1.0" } }, + "node_modules/urijs": { + "version": "1.19.11", + "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.11.tgz", + "integrity": "sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ==", + "license": "MIT" + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -13436,17 +13410,19 @@ } }, "node_modules/vite": { - "version": "7.1.5", - "resolved": "https://registry.npmjs.org/vite/-/vite-7.1.5.tgz", - "integrity": "sha512-4cKBO9wR75r0BeIWWWId9XK9Lj6La5X846Zw9dFfzMRw38IlTk2iCcUt6hsyiDRcPidc55ZParFYDXi0nXOeLQ==", + "name": "rolldown-vite", + "version": "7.1.16", + "resolved": "https://registry.npmjs.org/rolldown-vite/-/rolldown-vite-7.1.16.tgz", + "integrity": "sha512-cK6tCmZyEC0KRAcXTjQ+ara+wkqmaE7WUoI0ZfZzDuvaRaZ3mtvbhTJc4cH+PjKRok++++Z1bZZaNlf3+SnnGA==", "dev": true, "license": "MIT", "dependencies": { - "esbuild": "^0.25.0", + "@oxc-project/runtime": "0.92.0", "fdir": "^6.5.0", + "lightningcss": "^1.30.2", "picomatch": "^4.0.3", "postcss": "^8.5.6", - "rollup": "^4.43.0", + "rolldown": "1.0.0-beta.42", "tinyglobby": "^0.2.15" }, "bin": { @@ -13463,9 +13439,9 @@ }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", + "esbuild": "^0.25.0", "jiti": ">=1.21.0", "less": "^4.0.0", - "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", @@ -13478,15 +13454,15 @@ "@types/node": { "optional": true }, + "esbuild": { + "optional": true + }, "jiti": { "optional": true }, "less": { "optional": true }, - "lightningcss": { - "optional": true - }, "sass": { "optional": true }, @@ -13511,14 +13487,14 @@ } }, "node_modules/vite-plugin-mkcert": { - "version": "1.17.8", - "resolved": "https://registry.npmjs.org/vite-plugin-mkcert/-/vite-plugin-mkcert-1.17.8.tgz", - "integrity": "sha512-S+4tNEyGqdZQ3RLAG54ETeO2qyURHWrVjUWKYikLAbmhh/iJ+36gDEja4OWwFyXNuvyXcZwNt5TZZR9itPeG5Q==", + "version": "1.17.9", + "resolved": "https://registry.npmjs.org/vite-plugin-mkcert/-/vite-plugin-mkcert-1.17.9.tgz", + "integrity": "sha512-SwI7yqp2Cq4r2XItarnHRCj2uzHPqevbxFNMLpyN+LDXd5w1vmZeM4l5X/wCZoP4mjPQYN+9+4kmE6e3nPO5fg==", "dev": true, "license": "MIT", "dependencies": { - "axios": "^1.8.3", - "debug": "^4.4.0", + "axios": "^1.12.2", + "debug": "^4.4.3", "picocolors": "^1.1.1" }, "engines": { @@ -13581,15 +13557,15 @@ }, "node_modules/vscode": { "name": "@codingame/monaco-vscode-extension-api", - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-extension-api/-/monaco-vscode-extension-api-20.2.1.tgz", - "integrity": "sha512-K2VFVhQZUpBS+YJRr4DYgvNjelu7dWw81YWg8PwEVj7X8vSd9DqQZbTvudsiJDhlkkY4KWqZQjtITY0fc/X3QQ==", + "version": "21.6.0", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-extension-api/-/monaco-vscode-extension-api-21.6.0.tgz", + "integrity": "sha512-oslSpuCAZKS88hPx76Cickqd9/z5M1koUkx8UOqnfVIFAemnMHkZWWC4bOa/Q7wFZBy+1qB0/OuhrIBtycj5vA==", "license": "MIT", "dependencies": { - "@codingame/monaco-vscode-34a0ffd3-b9f5-5699-b43b-38af5732f38a-common": "20.2.1", - "@codingame/monaco-vscode-4a3ac544-9a61-534c-88df-756262793ef7-common": "20.2.1", - "@codingame/monaco-vscode-api": "20.2.1", - "@codingame/monaco-vscode-extensions-service-override": "20.2.1" + "@codingame/monaco-vscode-4a3ac544-9a61-534c-88df-756262793ef7-common": "21.6.0", + "@codingame/monaco-vscode-4bf376c2-03c7-58cb-8303-c67aeefa3d3d-common": "21.6.0", + "@codingame/monaco-vscode-api": "21.6.0", + "@codingame/monaco-vscode-extensions-service-override": "21.6.0" } }, "node_modules/vscode-jsonrpc": { @@ -13714,6 +13690,22 @@ "integrity": "sha512-RiMReJrTAiA+mBjGONMnjVDP2u3p9R1vkcGz6gDIrOMT3oGuYwX2WRMYI9ipkphSuE5XKEhydbhNEJh4NY9mlw==", "license": "Apache-2.0" }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/wheel": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/wheel/-/wheel-1.0.0.tgz", @@ -13767,9 +13759,9 @@ "integrity": "sha512-s+bdIgT/fA5em3zYUwF8D14uA/dZh7iu0krZYZQqZUO7txN37hwSCVfovbMkIwm4zPbsJ50mU8DRLt7UpAPZIw==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.512.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.512.0.tgz", - "integrity": "sha512-mOkuspfjPPhGZwmerBlFOjRKHjycrlzZUpxO7gHy5D6kwKI2bSz+VI3TsJqmK1y9PwAGgO8JnwRmNAR/YLUkzA==" + "version": "1.565.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.565.0.tgz", + "integrity": "sha512-fcAy7SkyrCML6YgnsQ5H3utpJFxLqWssGmY+hJAibdQtAYNRASsSQZl87A46nIFO8TpKKiWioTSxZDl/9Ovtrw==" }, "node_modules/windmill-parser-wasm-ruby": { "version": "1.526.1", @@ -13777,19 +13769,19 @@ "integrity": "sha512-rMBQA8s21wmL2kA5ztRs/ZgVA3ckxe9/NLjxl3iQPL0CX6DlvfaUH0O+AnhpXXDMyBs1Y1SZIhcnbnvsHZ3R8g==" }, "node_modules/windmill-parser-wasm-rust": { - "version": "1.510.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-rust/-/windmill-parser-wasm-rust-1.510.1.tgz", - "integrity": "sha512-tqT+w5gvwiX9NZCzT7iafh7dWrWSs46t+LI+N8/1+QOpv95G9/NJ/m0DIfH9Wyfkvtx0ge6igLMFha2wlPKG+w==" + "version": "1.558.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-rust/-/windmill-parser-wasm-rust-1.558.1.tgz", + "integrity": "sha512-21S7lm1KF8zO1187rbq14hzPHII2RdM2+D44MoAh1F6VoaScj+Puq0z5B1O/hwn/95R/a9jBlL2D8jbkXtlD1A==" }, "node_modules/windmill-parser-wasm-ts": { - "version": "1.538.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.538.0.tgz", - "integrity": "sha512-hHhMIVIPhmsHx0lsNCGMoIa7cDBFlVWhhd9j/5yOIq2sxwqg5sl5juQIIJGvuwg5umdsD0ChlSm2/uES78DLYg==" + "version": "1.565.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.565.0.tgz", + "integrity": "sha512-ui7dQ2kizWSG0ELpDU70Ccfk47EjC+yLI3tf6O+BdINs+YTjxA65MjShsvE3l4MnaxN8knbOLoSHkvIYCu77mg==" }, "node_modules/windmill-parser-wasm-yaml": { - "version": "1.510.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-yaml/-/windmill-parser-wasm-yaml-1.510.1.tgz", - "integrity": "sha512-zQ1imcKrhP3iccJ01BK0+tptguo3Xc+J5ku2lgrZ+YQdDcC2wjGb6gH+kvcsMXDE3WT4aRwV3nL+ecHa7WHSrw==" + "version": "1.561.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-yaml/-/windmill-parser-wasm-yaml-1.561.0.tgz", + "integrity": "sha512-UbyxsRxJ/QDE+RFjj8q6cMZqr57gxHXBM+W8VLXnQ8I79W5KI+FhKcNFraUpXzqQjalZJ3cVZXXr8C7cTlJ8IQ==" }, "node_modules/windmill-sql-datatype-parser-wasm": { "version": "1.512.0", @@ -13797,9 +13789,9 @@ "integrity": "sha512-uHNL8F72/Tf96xF3hOHnPDjkEyqXw7fNjcPJiUhth9sTQkcwUIoJMOdwm8/cs+j9kKVRJ4tgNYMHEBLylazp6g==" }, "node_modules/windmill-utils-internal": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.3.0.tgz", - "integrity": "sha512-UH7G+NVODkhm4o3BbjaOrSE2Qu+J6ro7+vpsIs+GvjDZM4ogSN7aJfnQNHW7Ke0VY74BKZVClTKROe/N6I5Reg==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.3.1.tgz", + "integrity": "sha512-afRGUDcvaUfGu7FA6DD0xWECQiKnXADs0N4WyQQ+OvaloxZ4oQzdEpLnVab/m3T02hhs29ru4Ilrdxu3ozyT5Q==", "license": "Apache 2.0" }, "node_modules/word-wrap": { @@ -13816,7 +13808,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", - "dev": true, "license": "MIT" }, "node_modules/wrap-ansi": { diff --git a/frontend/package.json b/frontend/package.json index 22ea609f18..e3efa90d53 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.542.1", + "version": "1.573.3", "scripts": { "dev": "vite dev", "build": "vite build", @@ -36,7 +36,6 @@ "@types/vscode": "^1.83.5", "@typescript-eslint/eslint-plugin": "^5.59.8", "@typescript-eslint/parser": "^5.60.0", - "@windmill-labs/esbuild-import-meta-url-plugin": "0.0.0-semantic-release", "@zerodevx/svelte-toast": "^0.9.6", "autoprefixer": "^10.4.13", "cssnano": "^6.0.1", @@ -65,7 +64,7 @@ "tar": "^7.4.3", "tslib": "^2.6.1", "typescript": "^5.5.0", - "vite": "^7.1.5", + "vite": "npm:rolldown-vite@latest", "vite-plugin-mkcert": "^1.17.5", "yootils": "^0.3.1" }, @@ -78,13 +77,12 @@ "dependencies": { "@anthropic-ai/sdk": "^0.60.0", "@aws-crypto/sha256-js": "^4.0.0", - "@codingame/monaco-vscode-configuration-service-override": "~20.2.1", - "@codingame/monaco-vscode-editor-api": "~20.2.1", - "@codingame/monaco-vscode-standalone-css-language-features": "~20.2.1", - "@codingame/monaco-vscode-standalone-html-language-features": "~20.2.1", - "@codingame/monaco-vscode-standalone-json-language-features": "~20.2.1", - "@codingame/monaco-vscode-standalone-languages": "~20.2.1", - "@codingame/monaco-vscode-standalone-typescript-language-features": "~20.2.1", + "@codingame/monaco-vscode-editor-api": "=21.6.0", + "@codingame/monaco-vscode-standalone-css-language-features": "=21.6.0", + "@codingame/monaco-vscode-standalone-html-language-features": "=21.6.0", + "@codingame/monaco-vscode-standalone-json-language-features": "=21.6.0", + "@codingame/monaco-vscode-standalone-languages": "=21.6.0", + "@codingame/monaco-vscode-standalone-typescript-language-features": "=21.6.0", "@json2csv/plainjs": "^7.0.6", "@leeoniya/ufuzzy": "^1.0.8", "@popperjs/core": "^2.11.6", @@ -117,10 +115,9 @@ "lru-cache": "^11.1.0", "lucide-svelte": "^0.540.0", "minimatch": "^10.0.1", - "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@~20.2.1", - "monaco-editor-wrapper": "6.12.0", + "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@=21.6.0", "monaco-graphql": "=1.6.0", - "monaco-languageclient": "9.11.0", + "monaco-languageclient": "10.1.0", "monaco-vim": "^0.4.1", "ol": "^7.4.0", "openai": "^5.16.0", @@ -128,6 +125,7 @@ "p-limit": "^6.1.0", "panzoom": "^9.4.3", "pdfjs-dist": "4.8.69", + "quicktype-core": "^23.2.6", "quill": "^1.3.7", "rehype-github-alerts": "^3.0.0", "rehype-raw": "^7.0.0", @@ -136,7 +134,7 @@ "svelte-exmarkdown": "^5.0.0", "svelte-infinite-loading": "^1.4.0", "tailwind-merge": "^1.13.2", - "vscode": "npm:@codingame/monaco-vscode-extension-api@~20.2.1", + "vscode": "npm:@codingame/monaco-vscode-extension-api@=21.6.0", "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.1.0", "vscode-ws-jsonrpc": "~3.5.0", @@ -146,13 +144,13 @@ "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.510.1", "windmill-parser-wasm-py": "1.538.0", - "windmill-parser-wasm-regex": "1.512.0", + "windmill-parser-wasm-regex": "1.565.0", "windmill-parser-wasm-ruby": "1.526.1", - "windmill-parser-wasm-rust": "1.510.1", - "windmill-parser-wasm-ts": "1.538.0", - "windmill-parser-wasm-yaml": "1.510.1", + "windmill-parser-wasm-rust": "1.558.1", + "windmill-parser-wasm-ts": "1.565.0", + "windmill-parser-wasm-yaml": "1.561.0", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.3.0", + "windmill-utils-internal": "^1.3.1", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", @@ -350,8 +348,8 @@ "default": "./package/gen/index.js" }, "./components/flows/flowStore": { - "types": "./package/components/flows/flowStore.d.ts", - "default": "./package/components/flows/flowStore.js" + "types": "./package/components/flows/flowStore.svelte.d.ts", + "default": "./package/components/flows/flowStore.svelte.js" }, "./components/icons": { "types": "./package/components/icons/index.d.ts", @@ -502,7 +500,7 @@ "./package/gen/index.d.ts" ], "components/flows/flowStore": [ - "./package/components/flows/flowStore.d.ts" + "./package/components/flows/flowStore.svelte.d.ts" ], "components/icons": [ "./package/components/icons/index.d.ts" @@ -543,4 +541,4 @@ "@rollup/rollup-linux-x64-gnu": "^4.35.0", "fsevents": "^2.3.3" } -} \ No newline at end of file +} diff --git a/frontend/scripts/untar_ui_builder.js b/frontend/scripts/untar_ui_builder.js index f0ebbd8c87..843ee15580 100644 --- a/frontend/scripts/untar_ui_builder.js +++ b/frontend/scripts/untar_ui_builder.js @@ -20,7 +20,7 @@ console.log('Running postinstall for root project'); import { x } from 'tar' -const tarUrl = 'https://pub-06154ed168a24e73a86ab84db6bf15d8.r2.dev/ui_builder-d44b577.tar.gz' +const tarUrl = 'https://pub-06154ed168a24e73a86ab84db6bf15d8.r2.dev/ui_builder-a3f259c.tar.gz' const outputTarPath = path.join(process.cwd(), 'ui_builder.tar.gz') const extractTo = path.join(process.cwd(), 'static/ui_builder/') diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts new file mode 100644 index 0000000000..6b302d281f --- /dev/null +++ b/frontend/src/lib/aiStore.ts @@ -0,0 +1,121 @@ +import { writable, get } from 'svelte/store' +import { workspaceAIClients } from './components/copilot/lib' +import { type AIProviderModel, type AIProvider, WorkspaceService, type AIConfig } from './gen' +import { COPILOT_SESSION_MODEL_SETTING_NAME, COPILOT_SESSION_PROVIDER_SETTING_NAME } from './stores' +import { getLocalSetting } from './utils' + +const USER_CUSTOM_PROMPTS_KEY = 'userCustomAIPrompts' + +const sessionModel = getLocalSetting(COPILOT_SESSION_MODEL_SETTING_NAME) +const sessionProvider = getLocalSetting(COPILOT_SESSION_PROVIDER_SETTING_NAME) +export const copilotSessionModel = writable( + sessionModel && sessionProvider + ? { + model: sessionModel, + provider: sessionProvider as AIProvider + } + : undefined +) + + +export const copilotInfo = writable<{ + enabled: boolean + codeCompletionModel?: AIProviderModel + defaultModel?: AIProviderModel + aiModels: AIProviderModel[] + customPrompts?: Record + maxTokensPerModel?: Record +}>({ + enabled: false, + codeCompletionModel: undefined, + defaultModel: undefined, + aiModels: [], + customPrompts: {}, + maxTokensPerModel: {} +}) + +export async function loadCopilot(workspace: string) { + workspaceAIClients.init(workspace) + try { + const info = await WorkspaceService.getCopilotInfo({ workspace }) + setCopilotInfo(info) + } catch (err) { + setCopilotInfo({}) + console.error('Could not get copilot info', err) + } +} + +export function setCopilotInfo(aiConfig: AIConfig) { + if (Object.keys(aiConfig.providers ?? {}).length > 0) { + const aiModels = Object.entries(aiConfig.providers ?? {}).flatMap( + ([provider, providerConfig]) => + providerConfig.models.map((m) => ({ model: m, provider: provider as AIProvider })) + ) + + copilotSessionModel.update((model) => { + if ( + model && + !aiModels.some((m) => m.model === model.model && m.provider === model.provider) + ) { + return undefined + } + return model + }) + + copilotInfo.set({ + enabled: true, + codeCompletionModel: aiConfig.code_completion_model, + defaultModel: aiConfig.default_model, + aiModels: aiModels, + customPrompts: aiConfig.custom_prompts ?? {}, + maxTokensPerModel: aiConfig.max_tokens_per_model ?? {} + }) + } else { + copilotSessionModel.set(undefined) + + copilotInfo.set({ + enabled: false, + codeCompletionModel: undefined, + defaultModel: undefined, + aiModels: [], + customPrompts: {}, + maxTokensPerModel: {} + }) + } +} + +export function getCurrentModel() { + const model = + get(copilotSessionModel) ?? get(copilotInfo).defaultModel ?? get(copilotInfo).aiModels[0] + if (!model) { + throw new Error('No model selected') + } + return model +} + +export function getUserCustomPrompts(): Record { + const stored = getLocalSetting(USER_CUSTOM_PROMPTS_KEY) + if (stored) { + try { + return JSON.parse(stored) + } catch (e) { + console.error('Failed to parse user custom prompts', e) + return {} + } + } + return {} +} + +export function getCombinedCustomPrompt(mode: string): string | undefined { + const workspacePrompt = get(copilotInfo).customPrompts?.[mode] + const userPrompts = getUserCustomPrompts() + const userPrompt = userPrompts[mode] + + const prompts = [workspacePrompt, userPrompt].filter((p) => p?.trim()) + + if (prompts.length === 0) { + return undefined + } + + return prompts.join('\n\n') +} diff --git a/frontend/src/lib/ansibleUtils.ts b/frontend/src/lib/ansibleUtils.ts new file mode 100644 index 0000000000..762c1793a2 --- /dev/null +++ b/frontend/src/lib/ansibleUtils.ts @@ -0,0 +1,384 @@ +interface DelegateToGitRepoConfig { + resource?: string + playbook?: string + inventories_location?: string +} + +/** + * Updates a specific field in the delegate_to_git_repo section + * @param code - The current YAML script content + * @param fieldName - The field name to update (resource, playbook, inventories_location) + * @param value - The value to set (or undefined to remove the field) + * @returns The modified YAML script content + */ +export function updateDelegateToGitRepoField(code: string, fieldName: string, value: string | undefined): string { + const lines = code.split('\n') + + // Find delegate_to_git_repo section + const delegateLineIndex = lines.findIndex(line => line.trim().startsWith('delegate_to_git_repo:')) + + if (delegateLineIndex === -1) { + // If no delegate section exists and we're setting a value, create the whole section + if (value !== undefined) { + return insertDelegateToGitRepoSection(code, { [fieldName]: value }) + } + return code + } + + // Find the specific field line + const fieldLineIndex = lines.findIndex((line, index) => + index > delegateLineIndex && line.trim().startsWith(`${fieldName}:`) + ) + + if (fieldLineIndex !== -1) { + if (value !== undefined) { + // Update existing field + lines[fieldLineIndex] = ` ${fieldName}: ${value}` + } else { + // Remove field + lines.splice(fieldLineIndex, 1) + } + } else if (value !== undefined) { + // Add new field after delegate_to_git_repo line + lines.splice(delegateLineIndex + 1, 0, ` ${fieldName}: ${value}`) + } + + return lines.join('\n') +} + +/** + * Inserts or updates multiple fields in a delegate_to_git_repo section + * @param code - The current YAML script content + * @param config - Configuration object with fields to update + * @returns The modified YAML script content + */ +export function updateDelegateToGitRepoConfig(code: string, config: DelegateToGitRepoConfig): string { + let updatedCode = code + + // Update each field that's provided + for (const [fieldName, value] of Object.entries(config)) { + if (value !== undefined) { + updatedCode = updateDelegateToGitRepoField(updatedCode, fieldName, value) + } + } + + return updatedCode +} + +/** + * Legacy function for backward compatibility + * Inserts or updates a delegate_to_git_repo section in an Ansible YAML script + * @param code - The current YAML script content + * @param resourcePath - The git repository resource path to delegate to + * @returns The modified YAML script content + */ +export function insertDelegateToGitRepoInCode(code: string, resourcePath: string): string { + return updateDelegateToGitRepoField(code, 'resource', resourcePath) +} + +/** + * Inserts a new delegate_to_git_repo section with the given configuration + * @param code - The current YAML script content + * @param config - Configuration object with fields to set + * @returns The modified YAML script content + */ +function insertDelegateToGitRepoSection(code: string, config: DelegateToGitRepoConfig): string { + const lines = code.split('\n') + + // Build the delegate section with all provided fields + const delegateSection = ['delegate_to_git_repo:'] + + // Add fields in a consistent order + if (config.resource) { + delegateSection.push(` resource: ${config.resource}`) + } + if (config.playbook) { + delegateSection.push(` playbook: ${config.playbook}`) + } + if (config.inventories_location) { + delegateSection.push(` inventories_location: ${config.inventories_location}`) + } + + // Find a good insertion point (after ---, then after inventories if they exist, otherwise at the top) + let insertionIndex = 0 + + // First, skip whitespace and find document start marker --- + for (let i = 0; i < lines.length; i++) { + const trimmedLine = lines[i].trim() + if (trimmedLine === '---') { + insertionIndex = i + 1 // Start after the document marker + break + } else if (trimmedLine && !trimmedLine.startsWith('#')) { + // Hit non-comment, non-whitespace content without finding ---, stop looking + break + } + } + + // Look for the end of inventories section + for (let i = insertionIndex; i < lines.length; i++) { + const line = lines[i].trim() + if (line.startsWith('inventories:')) { + // Find the end of inventories section + for (let j = i + 1; j < lines.length; j++) { + const nextLine = lines[j].trim() + if (nextLine && !nextLine.startsWith('-') && !nextLine.startsWith(' ') && !nextLine.startsWith('#')) { + insertionIndex = j + break + } + } + break + } else if (line && !line.startsWith('#') && insertionIndex <= 1) { + // First non-comment line after ---, insert before it + insertionIndex = i + break + } + } + + // Insert the delegate section + lines.splice(insertionIndex, 0, ...delegateSection, '') + + return lines.join('\n') +} + +/** + * Generic function to extract a specific field from delegate_to_git_repo section + * @param code - The YAML script content + * @param fieldName - The field name to extract (resource, playbook, inventories_location) + * @returns The field value if found, undefined otherwise + */ +function extractDelegateToGitRepoField(code: string, fieldName: string): string | undefined { + const lines = code.split('\n') + + // Find delegate_to_git_repo section + const delegateLineIndex = lines.findIndex(line => line.trim().startsWith('delegate_to_git_repo:')) + + if (delegateLineIndex === -1) { + return undefined + } + + // Look for the field line after delegate_to_git_repo + for (let i = delegateLineIndex + 1; i < lines.length; i++) { + const line = lines[i].trim() + if (line.startsWith(`${fieldName}:`)) { + // Extract the field value (everything after "fieldName:") + const fieldMatch = line.match(new RegExp(`^${fieldName}:\\s*(.+)$`)) + return fieldMatch?.[1]?.trim() + } else if (line && !line.startsWith(' ') && !line.startsWith('\t')) { + // Hit a new top-level section, stop looking + break + } + } + + return undefined +} + +/** + * Extracts the current git repository resource from delegate_to_git_repo section + * @param code - The YAML script content + * @returns The resource path if found, undefined otherwise + */ +export function extractCurrentGitRepoResource(code: string): string | undefined { + return extractDelegateToGitRepoField(code, 'resource') +} + +/** + * Extracts the current playbook path from delegate_to_git_repo section + * @param code - The YAML script content + * @returns The playbook path if found, undefined otherwise + */ +export function extractDelegateToGitRepoPlaybook(code: string): string | undefined { + return extractDelegateToGitRepoField(code, 'playbook') +} + +/** + * Extracts the current inventories location from delegate_to_git_repo section + * @param code - The YAML script content + * @returns The inventories location if found, undefined otherwise + */ +export function extractDelegateToGitRepoInventoriesLocation(code: string): string | undefined { + return extractDelegateToGitRepoField(code, 'inventories_location') +} + +/** + * Extracts all delegate_to_git_repo configuration from the code + * @param code - The YAML script content + * @returns Configuration object with all extracted fields + */ +export function extractDelegateToGitRepoConfig(code: string): DelegateToGitRepoConfig { + return { + resource: extractCurrentGitRepoResource(code), + playbook: extractDelegateToGitRepoPlaybook(code), + inventories_location: extractDelegateToGitRepoInventoriesLocation(code) + } +} + +/** + * Inserts or updates additional_inventories section in an Ansible YAML script + * @param code - The current YAML script content + * @param inventoryPaths - Array of inventory file paths + * @returns The modified YAML script content + */ +export function insertAdditionalInventories(code: string, inventoryPaths: string[]): string { + const lines = code.split('\n') + + // Find and update existing additional_inventories section if it exists + const additionalInventoriesIndex = lines.findIndex(line => line.trim().startsWith('additional_inventories:')) + if (additionalInventoriesIndex !== -1) { + // Determine the indentation level of the additional_inventories line + const sectionLine = lines[additionalInventoriesIndex] + const sectionIndentation = sectionLine.length - sectionLine.trimStart().length + + // Find the options: field within the section + let optionsIndex = -1 + let optionsEndIndex = -1 + + for (let i = additionalInventoriesIndex + 1; i < lines.length; i++) { + const line = lines[i] + const trimmedLine = line.trim() + + // Skip empty lines + if (!trimmedLine) { + continue + } + + // Calculate indentation of current line + const currentIndentation = line.length - line.trimStart().length + + // If we find a line with same or lesser indentation than the section header, + // we've reached the end of the additional_inventories section + if (currentIndentation <= sectionIndentation) { + break + } + + // Look for options: field (should be directly under additional_inventories) + if (trimmedLine.startsWith('- options:') && currentIndentation > sectionIndentation) { + optionsIndex = i + + // Check if it's inline format: options: [...] + if (trimmedLine.includes('[') && trimmedLine.includes(']')) { + // Inline format - just this line + optionsEndIndex = i + 1 + break + } else { + // Dash format - find all the dash items + optionsEndIndex = i + 1 + for (let j = i + 1; j < lines.length; j++) { + const nextLine = lines[j] + const nextTrimmed = nextLine.trim() + const nextIndentation = nextLine.length - nextLine.trimStart().length + + // Skip empty lines + if (!nextTrimmed) { + continue + } + + // If we hit a line that's not more indented than options:, we're done + if (nextIndentation <= currentIndentation) { + break + } + + // If it's a dash item, include it + if (nextTrimmed.startsWith('-')) { + optionsEndIndex = j + 1 + } else { + // Hit a non-dash line that's indented - stop here + break + } + } + break + } + } + } + + // Format the new options content + const optionsIndentation = ' ' // Standard 2-space indentation under additional_inventories + const formattedPaths = inventoryPaths.map(path => `"delegated_git_repository/${path}"`) + const inlineFormat = `${optionsIndentation}- options: [${formattedPaths.join(', ')}]` + + let newOptionsContent: string[] + if (inlineFormat.length <= 100) { + // Use inline format + newOptionsContent = [inlineFormat] + } else { + // Use dash format + newOptionsContent = [`${optionsIndentation}- options:`] + inventoryPaths.forEach(path => { + newOptionsContent.push(`${optionsIndentation} - "delegated_git_repository/${path}"`) + }) + } + + if (optionsIndex !== -1) { + // Replace existing options: field + lines.splice(optionsIndex, optionsEndIndex - optionsIndex, ...newOptionsContent) + } else { + // Add options: field to existing section (right after the section header) + lines.splice(additionalInventoriesIndex + 1, 0, ...newOptionsContent) + } + + return lines.join('\n') + } + + // Format the inventory paths based on length + const formattedPaths = inventoryPaths.map(path => `"delegated_git_repository/${path}"`) + const inlineFormat = `options: [${formattedPaths.join(', ')}]` + + let inventorySection: string[] + if (inlineFormat.length <= 100) { + // Use inline format + inventorySection = [ + 'additional_inventories:', + ` ${inlineFormat}` + ] + } else { + // Use dash format with each item on new line + inventorySection = [ + 'additional_inventories:', + ' - options:' + ] + inventoryPaths.forEach(path => { + inventorySection.push(` - "delegated_git_repository/${path}"`) + }) + } + + // Find insertion point (after the complete delegate_to_git_repo section) + const delegateLineIndex = lines.findIndex(line => line.trim().startsWith('delegate_to_git_repo:')) + if (delegateLineIndex === -1) { + // If no delegate_to_git_repo section, insert at the beginning (after document marker if exists) + let insertionIndex = 0 + for (let i = 0; i < lines.length; i++) { + const trimmedLine = lines[i].trim() + if (trimmedLine === '---') { + insertionIndex = i + 1 + break + } else if (trimmedLine && !trimmedLine.startsWith('#')) { + break + } + } + lines.splice(insertionIndex, 0, ...inventorySection, '') + } else { + // Find the last actual content line of the delegate_to_git_repo section + let lastContentIndex = delegateLineIndex + for (let i = delegateLineIndex + 1; i < lines.length; i++) { + const line = lines[i] + const trimmedLine = line.trim() + + // If we hit a non-empty line that doesn't start with whitespace (not indented), + // we've reached the end of the delegate_to_git_repo section + if (trimmedLine && !line.startsWith(' ') && !line.startsWith('\t')) { + break + } + + // If this is an indented non-empty line, it's part of delegate_to_git_repo + if (trimmedLine && (line.startsWith(' ') || line.startsWith('\t'))) { + lastContentIndex = i + } + } + + // Insert right after the last content line of delegate_to_git_repo + const insertionIndex = lastContentIndex + 1 + lines.splice(insertionIndex, 0, ...inventorySection, '') + } + + return lines.join('\n') +} + diff --git a/frontend/src/lib/assets/app.css b/frontend/src/lib/assets/app.css index 6e0623940d..cdccc1a865 100644 --- a/frontend/src/lib/assets/app.css +++ b/frontend/src/lib/assets/app.css @@ -3,6 +3,12 @@ @tailwind components; @tailwind utilities; +@media (min-width: 1760px) { + :root { + font-size: 18px; + } +} + @layer base { /* Light mode: default border color */ @@ -10,7 +16,7 @@ .border-x, .border-y, .divide-x > :not([hidden]) ~ :not([hidden]), .divide-y > :not([hidden]) ~ :not([hidden]) { - border-color: #e5e7eb; /* gray-200 */ + border-color: rgb(var(--color-border-light)); } /* Dark mode: change border color */ @@ -18,7 +24,7 @@ .dark .border-x, .dark .border-y, .dark .divide-x > :not([hidden]) ~ :not([hidden]), .dark .divide-y > :not([hidden]) ~ :not([hidden]) { - border-color: #4b5563; /* gray-700 */ + border-color: rgb(var(--color-border-light)); } /* Chrome, Edge, and Safari */ @@ -56,6 +62,9 @@ list-style-type: '- '; padding-left: 3rem; } + .prose a { + @apply text-accent no-underline; + } .autocomplete-list-item-create { @apply !text-primary-inverse !bg-surface-inverse; @@ -193,3 +202,10 @@ svelte-virtual-list-contents > * + * { ); } +/* Prevent clock icon in input[type="time"] making the input taller */ + +/* Chrome, Safari, Edge, Opera */ +input[type="time"]::-webkit-calendar-picker-indicator { + margin: 0; + padding: 0; +} \ No newline at end of file diff --git a/frontend/src/lib/assets/tokens/README.md b/frontend/src/lib/assets/tokens/README.md new file mode 100644 index 0000000000..182966b886 --- /dev/null +++ b/frontend/src/lib/assets/tokens/README.md @@ -0,0 +1 @@ +The content of tokens.json is auto-generated by the 'Figma Variables to JSON' extension diff --git a/frontend/src/lib/assets/tokens/colorTokensConfig.ts b/frontend/src/lib/assets/tokens/colorTokensConfig.ts new file mode 100644 index 0000000000..619272a469 --- /dev/null +++ b/frontend/src/lib/assets/tokens/colorTokensConfig.ts @@ -0,0 +1,2 @@ +export const lightModeName = 'light' as const +export const darkModeName = 'dark-3' as const diff --git a/frontend/src/lib/assets/tokens/tokens.json b/frontend/src/lib/assets/tokens/tokens.json new file mode 100644 index 0000000000..f04c2432f3 --- /dev/null +++ b/frontend/src/lib/assets/tokens/tokens.json @@ -0,0 +1,448 @@ +{ + "tokens": { + "light": { + "surface-accent-primary": "#758ff8", + "surface-accent-hover": "#5074f6", + "surface-accent-clicked": "#2c5beb", + "text-primary": "#3d4758", + "text-secondary": "#718096", + "text-primary-inverse": "#f3f6f8", + "text-secondary-inverse": "#d3d6d8", + "text-tertiary-inverse": "#a8a9ac", + "surface-selected": "#ffffff", + "surface-disabled": "#d8d8e433", + "surface-secondary": "#efeff4", + "surface-hover": "#cfcfe233", + "surface-primary": "#fbfbfd", + "border-light": "#e5e7eb", + "border-normal": "#9ca3af", + "border-accent": "#2c5beb", + "surface-accent-selected": "#bfdbfe4c", + "surface-accent-secondary": "#293676", + "surface-tertiary": "#ffffff", + "text-emphasis": "#1d2430", + "text-hint": "#8d93a1", + "text-disabled": "#a0aec0", + "surface-accent-secondary-hover": "#1e255f", + "surface-accent-secondary-clicked": "#303f82", + "component-button-accent-secondary": "#ffffff", + "text-emphasis-inverse": "#f3f4f6", + "reserved-ai": "#a02cde", + "component-virtual-node": "#dce0f1", + "text-accent": "#2652df", + "border-selected": "#a0affa", + "surface-sunken": "#e8e8ef", + "text-tertiary": "#505c70", + "surface-input": "#ffffff" + }, + "dark": { + "surface-accent-primary": "#758ff8", + "surface-accent-hover": "#5074f6", + "surface-accent-clicked": "#2c5beb", + "text-primary": "#d4d7dd", + "text-secondary": "#aab0bb", + "text-primary-inverse": "#2d3748", + "text-secondary-inverse": "#666e7b", + "text-tertiary-inverse": "#a4a9b2", + "surface-selected": "#434c5e", + "surface-disabled": "#21273266", + "surface-secondary": "#2e3440", + "surface-hover": "#454f64", + "surface-primary": "#3b4252", + "border-light": "#485971", + "border-normal": "#718096", + "border-accent": "#a0affa", + "surface-accent-selected": "#6790c34c", + "surface-accent-secondary": "#e8ebfb", + "surface-tertiary": "#434c5e", + "text-emphasis": "#f3f4f6", + "text-hint": "#b3bac8", + "text-disabled": "#717b88", + "surface-accent-secondary-hover": "#c3c9df", + "surface-accent-secondary-clicked": "#9da6ca", + "component-button-accent-secondary": "#f3f4f6", + "text-emphasis-inverse": "#2d3748", + "reserved-ai": "#f0c6fb", + "component-virtual-node": "#4c566a", + "text-accent": "#c7cefc", + "border-selected": "#758ff8", + "surface-sunken": "#2e3440", + "text-tertiary": "#a8aeb7", + "surface-input": "#2e3440" + }, + "dark-3": { + "surface-accent-primary": "#7085db", + "surface-accent-hover": "#5670d5", + "surface-accent-clicked": "#425bbd", + "text-primary": "#d4d7dd", + "text-secondary": "#a9b0ba", + "text-primary-inverse": "#2d3748", + "text-secondary-inverse": "#666e7b", + "text-tertiary-inverse": "#a4a9b2", + "surface-selected": "#434c5e", + "surface-disabled": "#212732", + "surface-secondary": "#272c35", + "surface-hover": "#7784a119", + "surface-primary": "#2e3441", + "border-light": "#374457", + "border-normal": "#a9b0ba", + "border-accent": "#a0affa", + "surface-accent-selected": "#6790c44c", + "surface-accent-secondary": "#e8ebfb", + "surface-tertiary": "#353c4a", + "text-emphasis": "#eeeff2", + "text-hint": "#8d93a1", + "text-disabled": "#9098a2", + "surface-accent-secondary-hover": "#c3c9df", + "surface-accent-secondary-clicked": "#9da6ca", + "component-button-accent-secondary": "#f3f4f6", + "text-emphasis-inverse": "#2d3748", + "reserved-ai": "#f0c6fb", + "component-virtual-node": "#4c566a", + "text-accent": "#c7cefc", + "border-selected": "#6475b7", + "surface-sunken": "#242832", + "text-tertiary": "#a8aeb7", + "surface-input": "#292e38" + } + }, + "primitives": { + "light": { + "light-blue": "#bcd4fc", + "blue": "#5e81ac", + "dark-blue": "#394a6d", + "accent-blue": "#2c5beb", + "transparent": "#ffffff00", + "deep-blue-900": "#1e255f", + "deep-blue-800": "#293676", + "deep-blue-700": "#303f82", + "deep-blue-600": "#39498e", + "deep-blue-500": "#3f5097", + "deep-blue-400": "#5b6aa5", + "deep-blue-300": "#7784b4", + "deep-blue-200": "#9da6ca", + "deep-blue-100": "#c3c9df", + "deep-blue-50": "#e7eaf2", + "blue-900": "#1e3a8a", + "blue-800": "#183faf", + "blue-700": "#1847d2", + "blue-600": "#2652df", + "blue-500": "#2c5beb", + "blue-400": "#5074f6", + "blue-300": "#758ff8", + "blue-200": "#a0affa", + "blue-100": "#c7cefc", + "blue-50": "#e9ecfe", + "nord-0": "#2e3440", + "nord-1": "#3b4252", + "nord-2": "#434c5e", + "nord-3": "#4c566a", + "nord-5": "#e5e9f0", + "nord-6": "#eceff4", + "nord-7": "#8fbcbb", + "nord-8": "#88c0d0", + "nord-9": "#81a1c1", + "nord-10": "#5e81ac", + "nord-4": "#d8dee9", + "nord-11": "#bf616a", + "nord-12": "#d08770", + "nord-13": "#ebcb8b", + "nord-14": "#a3be8c", + "nord-15": "#b48ead", + "red-800": "#693237", + "red-600": "#aa3e47", + "red-500": "#d34f5a", + "red-400": "#f87171", + "red-200": "#fecaca", + "red-950": "#392b31", + "red-100": "#fee2e2", + "red-900": "#4c2d32", + "red-700": "#81383f", + "red-300": "#fca5a5", + "red-50": "#fef2f2", + "green-50": "#f0fdf4", + "green-100": "#dcfce7", + "green-200": "#bbf7d0", + "green-300": "#96f2b7", + "green-400": "#62e993", + "green-500": "#32c76d", + "green-600": "#319e69", + "green-700": "#3b7a5b", + "green-800": "#3b6054", + "green-900": "#284945", + "green-950": "#263b3d", + "orange-50": "#fff7ed", + "orange-100": "#ffedd5", + "orange-200": "#fed7aa", + "orange-300": "#fdba74", + "orange-400": "#fb923c", + "orange-500": "#f0721b", + "orange-600": "#cd6230", + "orange-700": "#905035", + "orange-800": "#654338", + "orange-900": "#4b3531", + "orange-950": "#3c2c2ddb", + "purple-50": "#faf5ff", + "purple-100": "#f3e8ff", + "purple-200": "#e5cdff", + "purple-300": "#cfa5fc", + "purple-400": "#b267fd", + "purple-500": "#9939f5", + "purple-600": "#8143ba", + "purple-700": "#5f407d", + "purple-800": "#483c60", + "purple-900": "#3a3549", + "purple-950": "#31313f", + "blue-950": "#213263" + } + }, + "guidelines": { "mode-1": { "blue": "#5e81ac", "demo-background": "#ffffff00" } }, + "tailwind-c-s-s-v-3-3-2": { + "mode-1": { + "black": "#000000", + "white": "#ffffff", + "slate-50": "#f8fafc", + "slate-100": "#f1f5f9", + "slate-200": "#e2e8f0", + "slate-300": "#cbd5e1", + "slate-400": "#94a3b8", + "slate-500": "#64748b", + "slate-600": "#475569", + "slate-700": "#334155", + "slate-800": "#1e293b", + "slate-900": "#0f172a", + "slate-950": "#020617", + "gray-50": "#f9fafb", + "gray-100": "#f3f4f6", + "gray-200": "#e5e7eb", + "gray-300": "#d1d5db", + "gray-400": "#9ca3af", + "gray-500": "#6b7280", + "gray-600": "#4b5563", + "gray-700": "#374151", + "gray-800": "#1f2937", + "gray-900": "#111827", + "gray-950": "#030712", + "zinc-50": "#fafafa", + "zinc-100": "#f4f4f5", + "zinc-200": "#e4e4e7", + "zinc-300": "#d4d4d8", + "zinc-400": "#a1a1aa", + "zinc-500": "#71717a", + "zinc-600": "#52525b", + "zinc-700": "#3f3f46", + "zinc-800": "#27272a", + "zinc-900": "#18181b", + "zinc-950": "#09090b", + "neutral-50": "#fafafa", + "neutral-100": "#f5f5f5", + "neutral-200": "#e5e5e5", + "neutral-300": "#d4d4d4", + "neutral-400": "#a3a3a3", + "neutral-500": "#737373", + "neutral-600": "#525252", + "neutral-700": "#404040", + "neutral-800": "#262626", + "neutral-900": "#171717", + "neutral-950": "#0a0a0a", + "stone-50": "#fafaf9", + "stone-100": "#f5f5f4", + "stone-200": "#e7e5e4", + "stone-300": "#d6d3d1", + "stone-400": "#a8a29e", + "stone-500": "#78716c", + "stone-600": "#57534e", + "stone-700": "#44403c", + "stone-800": "#292524", + "stone-900": "#1c1917", + "stone-950": "#0c0a09", + "red-50": "#fef2f2", + "red-100": "#fee2e2", + "red-200": "#fecaca", + "red-300": "#fca5a5", + "red-400": "#f87171", + "red-500": "#ef4444", + "red-600": "#dc2626", + "red-700": "#b91c1c", + "red-800": "#991b1b", + "red-900": "#7f1d1d", + "red-950": "#450a0a", + "orange-50": "#fff7ed", + "orange-100": "#ffedd5", + "orange-200": "#fed7aa", + "orange-300": "#fdba74", + "orange-400": "#fb923c", + "orange-500": "#f97316", + "orange-600": "#ea580c", + "orange-700": "#c2410c", + "orange-800": "#9a3412", + "orange-900": "#7c2d12", + "orange-950": "#431407", + "amber-50": "#fffbeb", + "amber-100": "#fef3c7", + "amber-200": "#fde68a", + "amber-300": "#fcd34d", + "amber-400": "#fbbf24", + "amber-500": "#f59e0b", + "amber-600": "#d97706", + "amber-700": "#b45309", + "amber-800": "#92400e", + "amber-900": "#78350f", + "amber-950": "#451a03", + "yellow-50": "#fefce8", + "yellow-100": "#fef9c3", + "yellow-200": "#fef08a", + "yellow-300": "#fde047", + "yellow-400": "#facc15", + "yellow-500": "#eab308", + "yellow-600": "#ca8a04", + "yellow-700": "#a16207", + "yellow-800": "#854d0e", + "yellow-900": "#713f12", + "yellow-950": "#422006", + "lime-50": "#f7fee7", + "lime-100": "#ecfccb", + "lime-200": "#d9f99d", + "lime-300": "#bef264", + "lime-400": "#a3e635", + "lime-500": "#84cc16", + "lime-600": "#65a30d", + "lime-700": "#4d7c0f", + "lime-800": "#3f6212", + "lime-900": "#365314", + "lime-950": "#1a2e05", + "green-50": "#f0fdf4", + "green-100": "#dcfce7", + "green-200": "#bbf7d0", + "green-300": "#86efac", + "green-400": "#4ade80", + "green-500": "#22c55e", + "green-600": "#16a34a", + "green-700": "#15803d", + "green-800": "#166534", + "green-900": "#14532d", + "green-950": "#052e16", + "emerald-50": "#ecfdf5", + "emerald-100": "#d1fae5", + "emerald-200": "#a7f3d0", + "emerald-300": "#6ee7b7", + "emerald-400": "#34d399", + "emerald-500": "#10b981", + "emerald-600": "#059669", + "emerald-700": "#047857", + "emerald-800": "#065f46", + "emerald-900": "#064e3b", + "emerald-950": "#022c22", + "teal-50": "#f0fdfa", + "teal-100": "#ccfbf1", + "teal-200": "#99f6e4", + "teal-300": "#5eead4", + "teal-400": "#2dd4bf", + "teal-500": "#14b8a6", + "teal-600": "#0d9488", + "teal-700": "#0f766e", + "teal-800": "#115e59", + "teal-900": "#134e4a", + "teal-950": "#042f2e", + "cyan-50": "#ecfeff", + "cyan-100": "#cffafe", + "cyan-200": "#a5f3fc", + "cyan-300": "#67e8f9", + "cyan-400": "#22d3ee", + "cyan-500": "#06b6d4", + "cyan-600": "#0891b2", + "cyan-700": "#0e7490", + "cyan-800": "#155e75", + "cyan-900": "#164e63", + "cyan-950": "#083344", + "sky-50": "#f0f9ff", + "sky-100": "#e0f2fe", + "sky-200": "#bae6fd", + "sky-300": "#7dd3fc", + "sky-400": "#38bdf8", + "sky-500": "#0ea5e9", + "sky-600": "#0284c7", + "sky-700": "#0369a1", + "sky-800": "#075985", + "sky-900": "#0c4a6e", + "sky-950": "#082f49", + "blue-50": "#eff6ff", + "blue-100": "#dbeafe", + "blue-200": "#bfdbfe", + "blue-300": "#93c5fd", + "blue-400": "#60a5fa", + "blue-500": "#3b82f6", + "blue-600": "#2563eb", + "blue-700": "#1d4ed8", + "blue-800": "#1e40af", + "blue-900": "#1e3a8a", + "blue-950": "#172554", + "indigo-50": "#eef2ff", + "indigo-100": "#e0e7ff", + "indigo-200": "#c7d2fe", + "indigo-300": "#a5b4fc", + "indigo-400": "#818cf8", + "indigo-500": "#6366f1", + "indigo-600": "#4f46e5", + "indigo-700": "#4338ca", + "indigo-800": "#3730a3", + "indigo-900": "#312e81", + "indigo-950": "#1e1b4b", + "violet-50": "#f5f3ff", + "violet-100": "#ede9fe", + "violet-200": "#ddd6fe", + "violet-300": "#c4b5fd", + "violet-400": "#a78bfa", + "violet-500": "#8b5cf6", + "violet-600": "#7c3aed", + "violet-700": "#6d28d9", + "violet-800": "#5b21b6", + "violet-900": "#4c1d95", + "violet-950": "#2e1065", + "purple-50": "#faf5ff", + "purple-100": "#f3e8ff", + "purple-200": "#e9d5ff", + "purple-300": "#d8b4fe", + "purple-400": "#c084fc", + "purple-500": "#a855f7", + "purple-600": "#9333ea", + "purple-700": "#7e22ce", + "purple-800": "#6b21a8", + "purple-900": "#581c87", + "purple-950": "#3b0764", + "fuchsia-50": "#fdf4ff", + "fuchsia-100": "#fae8ff", + "fuchsia-200": "#f5d0fe", + "fuchsia-300": "#f0abfc", + "fuchsia-400": "#e879f9", + "fuchsia-500": "#d946ef", + "fuchsia-600": "#c026d3", + "fuchsia-700": "#a21caf", + "fuchsia-800": "#86198f", + "fuchsia-900": "#701a75", + "fuchsia-950": "#4a044e", + "pink-50": "#fdf2f8", + "pink-100": "#fce7f3", + "pink-200": "#fbcfe8", + "pink-300": "#f9a8d4", + "pink-400": "#f472b6", + "pink-500": "#ec4899", + "pink-600": "#db2777", + "pink-700": "#be185d", + "pink-800": "#9d174d", + "pink-900": "#831843", + "pink-950": "#500724", + "rose-50": "#fff1f2", + "rose-100": "#ffe4e6", + "rose-200": "#fecdd3", + "rose-300": "#fda4af", + "rose-400": "#fb7185", + "rose-500": "#f43f5e", + "rose-600": "#e11d48", + "rose-700": "#be123c", + "rose-800": "#9f1239", + "rose-900": "#881337", + "rose-950": "#4c0519" + } + } +} diff --git a/frontend/src/lib/cloud.ts b/frontend/src/lib/cloud.ts index cc6319127a..a684153e7a 100644 --- a/frontend/src/lib/cloud.ts +++ b/frontend/src/lib/cloud.ts @@ -2,4 +2,4 @@ import { BROWSER } from 'esm-env' export function isCloudHosted(): boolean { return BROWSER && window.location.hostname == 'app.windmill.dev' -} +} \ No newline at end of file diff --git a/frontend/src/lib/common.ts b/frontend/src/lib/common.ts index b96af76d03..e5220a2655 100644 --- a/frontend/src/lib/common.ts +++ b/frontend/src/lib/common.ts @@ -1,4 +1,4 @@ -import type { Script } from './gen' +import type { Script, ScriptLang } from './gen' export type OwnerKind = 'group' | 'user' | 'folder' @@ -15,7 +15,7 @@ export interface PropertyDisplayInfo { propertiesNumber: number } -export type EnumType = string[] | undefined +export type EnumType = string[] | { value: string; label: string }[] | undefined export interface SchemaProperty { type: string | undefined @@ -49,6 +49,7 @@ export interface SchemaProperty { placeholder?: string oneOf?: SchemaProperty[] originalType?: string + disabled?: boolean } export interface ModalSchemaProperty { @@ -108,6 +109,8 @@ export function modalToSchema(schema: ModalSchemaProperty): SchemaProperty { export type Schema = { $schema: string | undefined type: string + "x-windmill-dyn-select-code"?: string + "x-windmill-dyn-select-lang"?: ScriptLang properties: { [name: string]: SchemaProperty } order?: string[] required: string[] diff --git a/frontend/src/lib/components/AIAgentLogViewer.svelte b/frontend/src/lib/components/AIAgentLogViewer.svelte index 2640ea52eb..1671fe7b77 100644 --- a/frontend/src/lib/components/AIAgentLogViewer.svelte +++ b/frontend/src/lib/components/AIAgentLogViewer.svelte @@ -11,16 +11,17 @@ import FlowLogViewerWrapper from './FlowLogViewerWrapper.svelte' import { z } from 'zod' import { onMount } from 'svelte' + import type { AgentTool } from './flows/agentToolUtils' type AgentActionWithContent = NonNullable[number] & { - content: string + content?: unknown } const resultSchema = z.object({ messages: z.array( z.object({ role: z.string(), - content: z.string().optional(), + content: z.unknown(), agent_action: z .union([ z.object({ @@ -29,6 +30,13 @@ module_id: z.string(), function_name: z.string() }), + z.object({ + type: z.literal('mcp_tool_call'), + call_id: z.string(), + function_name: z.string(), + resource_path: z.string(), + arguments: z.record(z.unknown()).optional() + }), z.object({ type: z.literal('message') }) @@ -39,7 +47,7 @@ }) interface Props { - tools: FlowModule[] + tools: AgentTool[] agentJob: Partial & Pick & { type: 'CompletedJob' } workspaceId?: string | undefined storedToolCallJobs?: Record @@ -69,6 +77,13 @@ job_id: toolCall.job_id } onToolJobLoaded?.(job, idx) + } else if (toolCall.type === 'mcp_tool_call') { + fakeModuleStates[idx.toString()] = { + type: 'Success', + args: toolCall.arguments ?? {}, + logs: '', + result: toolCall.content + } } else { fakeModuleStates[idx.toString()] = { type: 'Success', @@ -104,7 +119,15 @@ module_id: m.agent_action.module_id, function_name: m.agent_action.function_name } - : undefined) as AgentActionWithContent | undefined + : m.agent_action?.type === 'mcp_tool_call' + ? { + type: 'mcp_tool_call', + content: m.content, + call_id: m.agent_action.call_id, + function_name: m.agent_action.function_name, + arguments: m.agent_action.arguments + } + : undefined) as AgentActionWithContent | undefined ) .filter((m) => m !== undefined) @@ -122,13 +145,22 @@ type: 'identity' as const } } + } else if (toolCall.type === 'mcp_tool_call') { + return { + id: idx.toString(), + value: { + type: 'identity' as const + }, + summary: toolCall.function_name, + arguments: toolCall.arguments + } } else { const module = tools.find((m) => m.summary === toolCall.function_name) return module - ? { + ? ({ ...module, id: idx.toString() - } + } as FlowModule) : undefined } }) diff --git a/frontend/src/lib/components/AIProviderPicker.svelte b/frontend/src/lib/components/AIProviderPicker.svelte index 36226c9c34..821302887d 100644 --- a/frontend/src/lib/components/AIProviderPicker.svelte +++ b/frontend/src/lib/components/AIProviderPicker.svelte @@ -8,6 +8,8 @@ import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' import ResourcePicker from './ResourcePicker.svelte' + import ToggleButtonMore from './common/toggleButton-v2/ToggleButtonMore.svelte' + import Toggle from './Toggle.svelte' interface ProviderValue { kind?: AIProvider @@ -23,29 +25,48 @@ let { value = $bindable(), disabled = false, actions }: Props = $props() - // Initialize value if undefined - if (!value) { - const providers = Object.keys(AI_PROVIDERS) - value = { - kind: providers.length > 0 ? (providers[0] as AIProvider) : undefined, - resource: undefined, - model: undefined - } - } - let loading = $state(false) let availableModels = $state([]) let filterText = $state('') + let useAsDefault = $state(false) let modelsCache = new Map() + const STORAGE_KEY = 'windmill_ai_provider_config' + + // Initialize value if undefined + if (!value) { + const storedConfig = loadStoredConfig() + if (storedConfig) { + value = storedConfig + useAsDefault = true + } else { + const providers = Object.keys(AI_PROVIDERS) + value = { + kind: providers.length > 0 ? (providers[0] as AIProvider) : undefined, + resource: undefined, + model: undefined + } + useAsDefault = false + } + } else { + useAsDefault = isSameAsStoredConfig(value) + } + // Reactive items for the Select component - let items = $derived( - availableModels.map((model) => ({ + let items = $derived.by(() => { + const r = availableModels.map((model) => ({ label: model, value: model })) - ) + if (value?.model && !availableModels.find((model) => model === value.model)) { + r.push({ + label: value.model, + value: value.model + }) + } + return r + }) // Provider options for the toggle button group const providerOptions = Object.entries(AI_PROVIDERS).map(([key, details]) => ({ @@ -53,6 +74,61 @@ label: details.label })) + // Check if the current config is the same as the stored config + function isSameAsStoredConfig(config: ProviderValue): boolean { + const storedConfig = loadStoredConfig() + return ( + storedConfig !== undefined && + storedConfig?.kind === config.kind && + storedConfig?.resource === config.resource && + storedConfig?.model === config.model + ) + } + + // Load stored configuration from localStorage + function loadStoredConfig(): ProviderValue | undefined { + if (typeof localStorage === 'undefined') { + return undefined + } + try { + const stored = localStorage.getItem(STORAGE_KEY) + if (stored) { + const parsed = JSON.parse(stored) + // Validate that the stored provider is still available + if (parsed.kind && AI_PROVIDERS[parsed.kind]) { + return parsed + } + } + } catch (e) { + console.error('Failed to load AI provider config from localStorage:', e) + } + return undefined + } + + // Save configuration to localStorage + function saveConfig(config: ProviderValue) { + if (typeof localStorage === 'undefined') { + return + } + try { + localStorage.setItem(STORAGE_KEY, JSON.stringify(config)) + } catch (e) { + console.error('Failed to save AI provider config to localStorage:', e) + } + } + + // Remove configuration from localStorage + function removeConfig() { + if (typeof localStorage === 'undefined') { + return + } + try { + localStorage.removeItem(STORAGE_KEY) + } catch (e) { + console.error('Failed to remove AI provider config from localStorage:', e) + } + } + async function loadModels(signal?: AbortSignal) { const provider = value?.kind const resourceValue = value?.resource @@ -91,28 +167,6 @@ } } - // Reload models when provider or resourcePath changes - $effect(() => { - const abortController = new AbortController() - const provider = value?.kind - const resourceValue = value?.resource - const resourcePath = resourceValueToPath(resourceValue) - - filterText = '' - - if (provider && resourcePath) { - loadModels(abortController.signal) - } else { - const defaultModels = provider ? AI_PROVIDERS[provider]?.defaultModels || [] : [] - availableModels = defaultModels - loading = false - } - - return () => { - abortController.abort() - } - }) - // Handle provider selection function onProviderChange(selectedProvider: AIProvider) { if (value) { @@ -145,24 +199,63 @@ return `$res:${path}` } } + + // Reload models when provider or resourcePath changes + $effect(() => { + const abortController = new AbortController() + const provider = value?.kind + const resourceValue = value?.resource + const resourcePath = resourceValueToPath(resourceValue) + + filterText = '' + + if (provider && resourcePath) { + loadModels(abortController.signal) + } else { + const defaultModels = provider ? AI_PROVIDERS[provider]?.defaultModels || [] : [] + availableModels = defaultModels + loading = false + } + + return () => { + abortController.abort() + } + }) + + $effect(() => { + if (useAsDefault && value && value.kind && value.resource && value.model) { + saveConfig(value) + } + }) -
+
-
- - {#snippet children({ item })} - {#each providerOptions as option} - - {/each} - {/snippet} - -
+ + {#snippet children({ item })} + {#each providerOptions.slice(0, 3) as option} + + {/each} + p.value === value.kind) >= 3 ? '' : 'More'} + togglableItems={providerOptions.slice(3)} + {item} + bind:selected={() => value?.kind, (v) => v && onProviderChange(v)} + /> + {/snippet} + -
+
-

resource

+

resource

resourceValueToPath(value?.resource), @@ -181,18 +274,40 @@
-

model

+

model

+
+ {:else if description == undefined || description == ''} +
No description provided
+ {:else} + + {/if}
- - {#if renderDescription} -
-
GH Markdown
- -
- {:else if description == undefined || description == ''} -
No description provided
- {:else} -
- - {/if} -
+ {#key resourceTypeInfo} {:else if step == 2 && !manual} {#if manual == false && resourceType != ''} -

{resourceType}

-
Create a resource backed by an OAuth connection, whose token is fetched from the external - services and refreshed automatically if needed before expiration.
-

Description

-
- -
+
+
+

{resourceType}

+
Create a resource backed by an OAuth connection, whose token is fetched from the + external services and refreshed automatically if needed before expiration.
+
- {#if supportsClientCredentials} -
-

Authentication Method

-
- - Use Client Credentials Flow - - Server-to-server authentication without user interaction. -

- Provide your own OAuth client credentials for this resource. -
+ {#if resourceTypeInfo?.description} +
+

Description

+
+ +
+ {/if} - {#if useClientCredentials} -
- - - + {#if supportsClientCredentials} +
+

Authentication Method

+
+ + + + Server-to-server authentication without user interaction. +

+ Provide your own OAuth client credentials for this resource. +
+
+ + {#if useClientCredentials} +
+ + + +
+ {/if} +
+ {/if} + +
+

Scopes

+ + {#if editScopes} + + {:else} +
+ {#each scopes as scope} +
- {scope}
+ {/each}
{/if}
- {/if} - -

Scopes

- - {#if editScopes} - - {:else} -
- {#each scopes as scope} -
- {scope}
- {/each} -
- {/if} +
{/if} {:else if step == 3 && !manual && !express} {#if useClientCredentials} - Connecting with client credentials... + Connecting with client credentials... {:else} - Finish connection in popup window + Finish connection in popup window {/if} {:else} {#if apiTokenApps[resourceType] || !manual} -
    -
  • +
      +
    • 1. A secret variable containing the {apiTokenApps[resourceType]?.linkedSecret ?? 'token'} - {truncateRev(value, 5, '*****')} + {truncateRev(value, 5, '*****')} will be stored a - {path}. + {path}.
    • -
    • +
    • 2. The resource containing that token will be stored at the same path {path}{path}. The Variable and Resource will be "linked together", they will be deleted and renamed together.
    {#if step > 2} - + {/if} + {/snippet} {#if itemsType?.type == 'number'} - + +
    + {@render deleteItemBtn()} +
    {:else if itemsType?.type == 'string' && itemsType?.contentEncoding == 'base64'} - fileChanged(x, (val) => (value[i] = val))} + + fileChangedInner(x.detail?.[0], (val) => (value[i] = val))} multiple={false} /> + {@render deleteItemBtn()} {:else if itemsType?.type == 'object' && itemsType?.resourceType === undefined && itemsType?.properties === undefined && !(format?.startsWith('resource-') && resourceTypes?.includes(format.split('-')[1]))} {#await import('$lib/components/JsonEditor.svelte')} @@ -787,6 +869,7 @@ bind:value={value[i]} /> {/await} + {@render deleteItemBtn()} {:else if Array.isArray(itemsType?.enum)} + {@render deleteItemBtn()} {:else if (itemsType?.type == 'resource' && itemsType?.resourceType && resourceTypes?.includes(itemsType.resourceType)) || (format?.startsWith('resource-') && resourceTypes?.includes(format.split('-')[1]))} {@const resourceFormat = itemsType?.type == 'resource' && @@ -816,6 +900,7 @@ format={resourceFormat} defaultValue={undefined} /> + {@render deleteItemBtn()} {:else if itemsType?.type == 'resource'} {#await import('$lib/components/JsonEditor.svelte')} @@ -832,9 +917,11 @@ bind:value={value[i]} /> {/await} + {@render deleteItemBtn()} {:else if itemsType?.type === 'object' && itemsType?.properties}
    + {@render deleteItemBtn()} {:else} - + +
    + {@render deleteItemBtn()} +
    {/if} -
    {/if} {/each} @@ -869,7 +953,7 @@ {#if value.startsWith('$res:')} {@render resourceInput()} {:else} -
    +
    Invalid string value: "{value}", expected array. Click add item to turn it into an array.
    @@ -877,70 +961,68 @@ {/if} {/key}
    -
    - -
    + {/if}
-
- { - // Once the user has changed the input type, we should not change it back automatically - if (!hasIsListJsonChanged) { - hasIsListJsonChanged = true - } + {#if !displayHeader} +
+ { + // Once the user has changed the input type, we should not change it back automatically + if (!hasIsListJsonChanged) { + hasIsListJsonChanged = true + } - evalValueToRaw() - isListJson = !isListJson - }} - checked={isListJson} - textClass="text-secondary" - size="xs" - options={{ right: 'json' }} - /> -
+ evalValueToRaw() + isListJson = !isListJson + }} + checked={isListJson} + textClass="text-secondary" + size="xs" + options={{ left: 'json' }} + /> +
+ {/if}
{:else if inputCat == 'dynamic'} {:else if inputCat == 'resource-object' && resourceTypes == undefined} - Loading resource types... + Loading resource types... {:else if inputCat == 'resource-object' && (resourceTypes == undefined || (format && format?.split('-').length > 1 && resourceTypes.includes(format?.substring('resource-'.length))))} {:else if inputCat == 'object' || inputCat == 'resource-object' || isListJson} {#if oneOf && oneOf.length >= 2} -
+
{#if oneOf && oneOf.length >= 2} 0} {#key redraw} -
- {#if orderEditable} - ({ - properties: obj.properties ?? {}, - order: obj.order, - $schema: '', - required: obj.required ?? [], - type: 'object' - }), - () => { - dispatch('nestedChange') - } - } - bind:args={value} - hiddenArgs={[ - oneOf?.find((o) => Object.keys(o.properties ?? {}).includes('kind')) - ? 'kind' - : 'label' - ]} - on:reorder={(e) => { - if (oneOf && oneOf[objIdx]) { - const keys = e.detail - oneOf[objIdx].order = keys - } - }} - on:nestedChange - {shouldDispatchChanges} - /> - {:else} - ({ + properties: obj.properties ?? {}, order: obj.order, $schema: '', required: obj.required ?? [], type: 'object' - }} - bind:args={ - () => value, - (v) => { - value = { ...v, [tagKey]: oneOfSelected } - } - } - {shouldDispatchChanges} - on:change={() => { + }), + () => { dispatch('nestedChange') - }} - on:nestedChange - /> - {/if} -
+ } + } + bind:args={value} + hiddenArgs={[ + oneOf?.find((o) => Object.keys(o.properties ?? {}).includes('kind')) + ? 'kind' + : 'label' + ]} + on:reorder={(e) => { + if (oneOf && oneOf[objIdx]) { + const keys = e.detail + oneOf[objIdx].order = keys + } + }} + on:nestedChange + {shouldDispatchChanges} + /> + {:else} + value, + (v) => { + value = { ...v, [tagKey]: oneOfSelected } + } + } + {shouldDispatchChanges} + on:change={() => { + dispatch('nestedChange') + }} + on:nestedChange + /> + {/if} {/key} {:else if disabled} @@ -1102,9 +1184,10 @@ {/if}
{:else if properties && Object.keys(properties).length > 0 && inputCat !== 'list'} -
+
{#if orderEditable} {:else} {/await} {/if} - {#if inputCat == 'list'} -
+ {#if inputCat == 'list' && !displayHeader} +
{ isListJson = !isListJson @@ -1194,13 +1278,17 @@ checked={isListJson} textClass="text-secondary" size="xs" - options={{ right: 'json' }} + options={{ left: 'json' }} />
{/if} {:else if inputCat == 'enum'}
{ + lastValue = undefined + value = undefined + }} create={extra['disableCreate'] != true} {defaultValue} valid={valid ?? true} @@ -1220,6 +1308,8 @@ {:else if inputCat == 'date'} {#if format === 'date'} + {:else if format === 'naive-date-time'} + {:else} {/if} @@ -1250,19 +1340,15 @@
{/if} {:else if inputCat == 'base64'} -
- fileChanged(x, (val) => (value = val))} +
+ fileChangedInner(x.detail?.[0], (val) => (value = val))} multiple={false} /> {#if value?.length} -
File length: {value.length} base64 chars ({(value.length / 1024 / 1024).toFixed( - 2 - )}MB)
+
+ File length: {value.length} base64 chars ({(value.length / 1024 / 1024).toFixed(2)}MB) +
{/if}
{:else if inputCat == 'resource-string'} @@ -1325,27 +1411,30 @@ {disabled} class={twMerge( 'w-full', - valid - ? '' - : 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-3' + inputBaseClass, + inputSizeClasses.md, + inputBorderClass({ error: !!error }) )} placeholder={placeholder ?? defaultValue ?? ''} bind:value > {/key} - {#if !disabled && itemPicker && extra?.['disableVariablePicker'] != true} - - - {/if} + {/if} + {#if !disabled && itemPicker && extra?.['disableVariablePicker'] != true} + + {/if}
{@render variableInput()} @@ -1356,7 +1445,7 @@
{#if !compact || (error && error != '')} -
+
{#if disabled || error === ''}   {:else} @@ -1374,10 +1463,4 @@ -webkit-appearance: none !important; margin: 0; } - - /* Firefox */ - input[type='number'] { - -moz-appearance: textfield !important; - appearance: textfield !important; - } diff --git a/frontend/src/lib/components/ArrayTypeNarrowing.svelte b/frontend/src/lib/components/ArrayTypeNarrowing.svelte index 86b4fb6c10..064faa6950 100644 --- a/frontend/src/lib/components/ArrayTypeNarrowing.svelte +++ b/frontend/src/lib/components/ArrayTypeNarrowing.svelte @@ -10,6 +10,7 @@ import type { SchemaProperty } from '$lib/common' import Toggle from './Toggle.svelte' import { tick } from 'svelte' + import Select from './select/Select.svelte' interface Props { canEditResourceType?: boolean @@ -54,39 +55,44 @@ {#if canEditResourceType || originalType == 'string[]' || originalType == 'object[]'} {:else if itemsType?.resourceType}
diff --git a/frontend/src/lib/components/Auth0Setting.svelte b/frontend/src/lib/components/Auth0Setting.svelte index 1116ea0d27..00e683dcf4 100644 --- a/frontend/src/lib/components/Auth0Setting.svelte +++ b/frontend/src/lib/components/Auth0Setting.svelte @@ -44,7 +44,7 @@
-