fix

* react to merged pr

* fix

* fix

.aiderignore

@@ -0,0 +1,3 @@
+/*
+!/backend/
+!/frontend/

.github/workflows/aider-after-review.yaml

@@ -0,0 +1,170 @@
+name: Aider Auto-fix PR Review Change Requests
+
+on:
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  auto-fix-review:
+    if: github.event.review.state == 'changes_requested' && contains(github.event.pull_request.title, '[Aider PR]')
+    runs-on: ubicloud-standard-8
+    permissions:
+      contents: write
+      pull-requests: write
+    env:
+      GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      WINDMILL_TOKEN: ${{ secrets.WINDMILL_TOKEN }}
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@v2
+        with:
+          egress-policy: audit
+
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git User
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Checkout PR Branch
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          echo "PR review trigger: Checking out PR branch..."
+          PR_NUMBER=${{ github.event.pull_request.number }}
+          PR_HEAD_REF=$(gh pr view $PR_NUMBER --json headRefName -q .headRefName --repo $GITHUB_REPOSITORY)
+          if [[ -z "$PR_HEAD_REF" || "$PR_HEAD_REF" == "null" ]]; then
+             echo "::error::Could not determine PR head branch for PR #$PR_NUMBER via gh CLI."
+             exit 1
+          fi
+          echo "Checking out PR head branch: $PR_HEAD_REF for PR #$PR_NUMBER"
+          git fetch origin "refs/heads/${PR_HEAD_REF}:refs/remotes/origin/${PR_HEAD_REF}" --no-tags
+          git checkout "$PR_HEAD_REF"
+          echo "Successfully checked out branch $(git rev-parse --abbrev-ref HEAD)"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Aider and Dependencies
+        run: |
+          python -m pip install aider-install; aider-install
+          pip install -U google-generativeai
+          sudo apt-get update && sudo apt-get install -y jq
+
+      - name: Generate Prompt from Review
+        id: generate_prompt
+        shell: bash
+        run: |
+          mkdir -p .github/aider
+          PROMPT_FILE_PATH=".github/aider/review-prompt.txt"
+
+          # Get PR review body
+          REVIEW_BODY="${{ github.event.review.body }}"
+          PR_NUMBER="${{ github.event.pull_request.number }}"
+
+          # Get PR description for context NOT USED FOR NOW
+          # PR_DETAILS=$(gh pr view $PR_NUMBER --json title,body --repo $GITHUB_REPOSITORY)
+          # PR_TITLE=$(echo "$PR_DETAILS" | jq -r .title)
+          # PR_BODY=$(echo "$PR_DETAILS" | jq -r .body)
+
+          # Get all PR review comments
+          REVIEW_COMMENTS=$(gh pr view $PR_NUMBER --json reviews -q '.reviews[] | select(.state == "CHANGES_REQUESTED") | .body' --repo $GITHUB_REPOSITORY)
+          REVIEW_BODY_Q=$(printf '%q' "$REVIEW_BODY")
+
+          # Update query to get review comments from all review types, not just "CHANGES_REQUESTED"
+          ALL_REVIEW_COMMENTS=$(gh api \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/comments \
+            | jq '[.[] | {diff_hunk: .diff_hunk, path: .path, body: .body}]')
+
+          BASE_PROMPT="Fix the following issues in the PR based on the review feedback. The review body is prepended with REVIEW. The review comments are prepended with REVIEW_COMMENTS. The review body and comments are separated by a blank line."
+          printf "%s\nREVIEW:\n%s\nREVIEW_COMMENTS:\n%s" \
+            "$BASE_PROMPT" "$REVIEW_BODY_Q" "$ALL_REVIEW_COMMENTS" > "$PROMPT_FILE_PATH"
+          echo "PROMPT_FILE_PATH=$PROMPT_FILE_PATH" >> $GITHUB_OUTPUT
+
+      - name: Run Aider with review prompt
+        run: |
+          aider \
+            --read .cursor/rules/rust-best-practices.mdc \
+            --read .cursor/rules/svelte5-best-practices.mdc \
+            --model gemini/gemini-2.5-pro-preview-05-06 \
+            --message-file .github/aider/review-prompt.txt \
+            --yes \
+            --no-check-update \
+            --auto-commits \
+            --no-analytics \
+            --no-gitignore \
+            | tee .github/aider/aider-output.txt || true
+          echo "Aider command completed. Output saved to .github/aider/aider-output.txt"
+          # Check if there are any changes to commit
+          if [[ -z "$(git status --porcelain)" ]]; then
+            echo "No changes detected after running Aider."
+            exit 0
+          fi
+
+      - name: Clean up prompt file
+        if: always()
+        run: rm -f .github/aider/review-prompt.txt
+
+      - name: Commit and Push Changes
+        id: commit_and_push
+        if: ${{ success() }}
+        run: |
+          CURRENT_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD)
+          echo "Attempting to push changes to PR branch $CURRENT_BRANCH_NAME for PR #${{ github.event.pull_request.number }}"
+
+          # Pull latest changes to avoid rejection due to non-fast-forward
+          git pull origin $CURRENT_BRANCH_NAME
+
+          if git push origin $CURRENT_BRANCH_NAME; then
+            echo "Push to $CURRENT_BRANCH_NAME successful."
+            echo "CHANGES_APPLIED=true" >> $GITHUB_OUTPUT
+          else
+            echo "::warning::Push to PR branch $CURRENT_BRANCH_NAME failed."
+            echo "CHANGES_APPLIED=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Comment on PR
+        if: success()
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUM: ${{ github.event.pull_request.number }}
+        run: |
+          # Create comment body in a temporary file to avoid command line length limits
+          if [[ "${{ steps.commit_and_push.outputs.CHANGES_APPLIED }}" == "true" ]]; then
+            cat > /tmp/pr-comment.md << EOL
+          🤖 I've automatically addressed the feedback based on the review.
+
+          ## Aider Output
+          \`\`\`
+          $(cat .github/aider/aider-output.txt || echo 'No output available')
+          \`\`\`
+
+          Please review the changes and let me know if further adjustments are needed.
+          EOL
+          else
+            cat > /tmp/pr-comment.md << EOL
+          🤖 I attempted to address the review feedback, but no modifications were made.
+
+          ## Aider Output
+          \`\`\`
+          $(cat .github/aider/aider-output.txt || echo 'No output available')
+          \`\`\`
+
+          Please review the output and provide additional guidance if needed.
+          EOL
+          fi
+
+          # Use the file for comment body
+          gh pr comment $PR_NUM --body-file /tmp/pr-comment.md

.github/workflows/aider.yaml

@@ -0,0 +1,342 @@
+name: Aider Auto-fix issues and PR comments via external prompt
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  auto-fix:
+    runs-on: ubicloud-standard-8
+    if: |
+      github.event_name == 'issue_comment' &&
+      contains(github.event.comment.body, '/aider') &&
+      !contains(github.event.comment.user.login, '[bot]')
+    permissions:
+      contents: write
+      pull-requests: write
+      issues: write
+    env:
+      GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      WINDMILL_TOKEN: ${{ secrets.WINDMILL_TOKEN }}
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@v2
+        with:
+          egress-policy: audit
+
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git User
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Checkout PR Branch
+        if: github.event_name == 'issue_comment' && github.event.issue.pull_request
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          echo "Issue comment trigger: Checking out PR branch..."
+          PR_NUMBER=${{ github.event.issue.number }}
+          PR_HEAD_REF=$(gh pr view $PR_NUMBER --json headRefName -q .headRefName --repo $GITHUB_REPOSITORY)
+          if [[ -z "$PR_HEAD_REF" || "$PR_HEAD_REF" == "null" ]]; then
+             echo "::error::Could not determine PR head branch for PR #$PR_NUMBER via gh CLI."
+             exit 1
+          fi
+          echo "Checking out PR head branch: $PR_HEAD_REF for PR #$PR_NUMBER"
+          git fetch origin "refs/heads/${PR_HEAD_REF}:refs/remotes/origin/${PR_HEAD_REF}" --no-tags
+          git checkout "$PR_HEAD_REF"
+          echo "Successfully checked out branch $(git rev-parse --abbrev-ref HEAD)"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Aider and Dependencies
+        run: |
+          python -m pip install aider-install; aider-install
+          pip install -U google-generativeai
+          sudo apt-get update && sudo apt-get install -y jq
+
+      - name: Determine Prompt for Aider
+        id: determine_prompt
+        shell: bash
+        run: |
+          PROMPT_FILE_PATH=".github/aider/issue-prompt.txt"
+          mkdir -p .github/aider
+
+          # Determine if this is a PR comment or regular issue comment
+          if [[ ! -z "${{ github.event.issue.pull_request }}" ]]; then
+            echo "This is a comment on a Pull Request"
+            PR_NUMBER="${{ github.event.issue.number }}"
+            
+            # Get PR description to check for issue references
+            PR_BODY=$(gh pr view $PR_NUMBER --json body -q .body --repo $GITHUB_REPOSITORY)
+            
+            # Extract issue number from PR description (looking for #123 or "fixes #123" patterns)
+            REFERENCED_ISSUE=$(echo "$PR_BODY" | grep -oE "#[0-9]+" | grep -oE "[0-9]+" | head -1)
+            
+            if [[ ! -z "$REFERENCED_ISSUE" ]]; then
+              echo "Found referenced issue #$REFERENCED_ISSUE in PR description"
+              
+              # Fetch the referenced issue details
+              ISSUE_DETAILS=$(gh issue view $REFERENCED_ISSUE --json title,body --repo $GITHUB_REPOSITORY)
+              ISSUE_TITLE=$(echo "$ISSUE_DETAILS" | jq -r .title)
+              ISSUE_BODY=$(echo "$ISSUE_DETAILS" | jq -r .body)
+              
+              # Store raw comment body in a file first to avoid shell interpretation issues
+              echo '${{ github.event.comment.body }}' > /tmp/raw_comment.txt
+              RAW_COMMENT_BODY=$(cat /tmp/raw_comment.txt)
+              # Remove the /aider prefix and trim whitespace
+              COMMENT_CONTENT=$(echo "$RAW_COMMENT_BODY" | sed 's|^/aider||' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')
+              
+              echo "Sending issue content and PR comment to external API…"
+              
+              ISSUE_TITLE_Q=$(printf '%q' "$ISSUE_TITLE")
+              ISSUE_BODY_Q=$(printf '%q' "$ISSUE_BODY")
+              
+              JSON_PAYLOAD=$(jq -n \
+                --arg title "$ISSUE_TITLE_Q" \
+                --arg body "$ISSUE_BODY_Q" \
+                '{"body":{"issue_title":$title,"issue_body":$body}}')
+              
+              API_RESULT=$(curl -s -w "\n%{http_code}" \
+                -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run_wait_result/p/f/ai/quiet_script" \
+                -H "Content-Type: application/json" \
+                -H "Authorization: Bearer $WINDMILL_TOKEN" \
+                --data-binary "$JSON_PAYLOAD" \
+                --max-time 90)
+              
+              HTTP_CODE=$(echo "$API_RESULT" | tail -n1)
+              BODY=$(echo "$API_RESULT" | sed '$d')
+              
+              echo "$BODY" > /tmp/api_response.txt
+
+              BASE_PROMPT="Try to fix the following issue based on the instruction given by the user. The issue is prepended with the word ISSUE. The instruction is prepended with the word INSTRUCTION. The issue and instruction are separated by a blank line."
+              if [[ "$HTTP_CODE" -eq 200 ]]; then
+                PROCESSED_ISSUE_PROMPT=$(jq -r '.effective_body // empty' /tmp/api_response.txt)
+                if [[ -z "$PROCESSED_ISSUE_PROMPT" || "$PROCESSED_ISSUE_PROMPT" == "null" ]]; then
+                  PROCESSED_ISSUE_PROMPT=""
+                fi
+                printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \
+                  "$BASE_PROMPT" "$PROCESSED_ISSUE_PROMPT" "$COMMENT_CONTENT" > "$PROMPT_FILE_PATH"
+              else
+                echo "::warning::API call failed (HTTP $HTTP_CODE). Using PR comment with issue context."
+                printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \
+                  "$BASE_PROMPT" "$ISSUE_BODY_Q" "$COMMENT_CONTENT" > "$PROMPT_FILE_PATH"
+              fi
+              rm -f /tmp/api_response.txt
+            else
+              echo "No referenced issue found in PR description, using comment content only"
+              # Use comment content directly as with regular issue comments
+              echo '${{ github.event.comment.body }}' > /tmp/raw_comment.txt
+              RAW_COMMENT_BODY=$(cat /tmp/raw_comment.txt)
+              COMMENT_CONTENT=$(echo "$RAW_COMMENT_BODY" | sed 's|^/aider||' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')
+              
+              if [[ -z "$COMMENT_CONTENT" ]]; then
+                echo "::error::Comment with /aider provided, but no instruction found after it. Cannot proceed."
+                printf "Error: /aider command found but no instruction followed." > "$PROMPT_FILE_PATH"
+                exit 1
+              else
+                echo "Using comment content as prompt."
+                printf '%s' "$COMMENT_CONTENT" > "$PROMPT_FILE_PATH"
+              fi
+            fi
+          else
+            echo "This is a comment on a regular issue"
+            
+            # Fetch the issue details
+            ISSUE_NUMBER="${{ github.event.issue.number }}"
+            ISSUE_DETAILS=$(gh issue view $ISSUE_NUMBER --json title,body --repo $GITHUB_REPOSITORY)
+            ISSUE_TITLE=$(echo "$ISSUE_DETAILS" | jq -r .title)
+            ISSUE_BODY=$(echo "$ISSUE_DETAILS" | jq -r .body)
+            
+            # Store raw comment body in a file first to avoid shell interpretation issues
+            echo '${{ github.event.comment.body }}' > /tmp/raw_comment.txt
+            # Extract the command part safely
+            RAW_COMMENT_BODY=$(cat /tmp/raw_comment.txt)
+            # Remove the /aider prefix and trim whitespace
+            COMMENT_CONTENT=$(echo "$RAW_COMMENT_BODY" | sed 's|^/aider||' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')
+
+            if [[ -z "$COMMENT_CONTENT" ]]; then
+              echo "::error::Comment with /aider provided, but no instruction found after it. Cannot proceed."
+              printf "Error: /aider command found but no instruction followed." > "$PROMPT_FILE_PATH"
+              exit 1
+            else
+              echo "Sending issue content and issue comment to external API…"
+              
+              ISSUE_TITLE_Q=$(printf '%q' "$ISSUE_TITLE")
+              ISSUE_BODY_Q=$(printf '%q' "$ISSUE_BODY")
+              COMMENT_CONTENT_Q=$(printf '%q' "$COMMENT_CONTENT")
+              
+              JSON_PAYLOAD=$(jq -n \
+                --arg title "$ISSUE_TITLE_Q" \
+                --arg body "$ISSUE_BODY_Q" \
+                --arg comment "$COMMENT_CONTENT_Q" \
+                '{"body":{"issue_title":$title,"issue_body":$body,"issue_comment":$comment}}')
+              
+              API_RESULT=$(curl -s -w "\n%{http_code}" \
+                -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run_wait_result/p/f/ai/quiet_script" \
+                -H "Content-Type: application/json" \
+                -H "Authorization: Bearer $WINDMILL_TOKEN" \
+                --data-binary "$JSON_PAYLOAD" \
+                --max-time 90)
+              
+              HTTP_CODE=$(echo "$API_RESULT" | tail -n1)
+              BODY=$(echo "$API_RESULT" | sed '$d')
+              
+              echo "$BODY" > /tmp/api_response.txt
+              
+              BASE_PROMPT="Try to fix the following issue based on the instruction given by the user. The issue is prepended with the word ISSUE. The instruction is prepended with the word INSTRUCTION. The issue and instruction are separated by a blank line."
+              if [[ "$HTTP_CODE" -eq 200 ]]; then
+                PROCESSED_ISSUE_PROMPT=$(jq -r '.effective_body // empty' /tmp/api_response.txt)
+                if [[ -z "$PROCESSED_ISSUE_PROMPT" || "$PROCESSED_ISSUE_PROMPT" == "null" ]]; then
+                  PROCESSED_ISSUE_PROMPT=""
+                fi
+                printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \
+                  "$BASE_PROMPT" "$PROCESSED_ISSUE_PROMPT" "$COMMENT_CONTENT" > "$PROMPT_FILE_PATH"
+              else
+                echo "::warning::API call failed (HTTP $HTTP_CODE). Using PR comment with issue context."
+                printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \
+                  "$BASE_PROMPT" "$ISSUE_BODY_Q" "$COMMENT_CONTENT" > "$PROMPT_FILE_PATH"
+              fi
+
+              rm -f /tmp/api_response.txt
+            fi
+          fi
+          echo "Prompt determined and written to $PROMPT_FILE_PATH"
+          echo "PROMPT_FILE_PATH=$PROMPT_FILE_PATH" >> $GITHUB_OUTPUT
+
+      - name: Probe Chat for Relevant Files
+        id: probe_files
+        env:
+          PROMPT_CONTENT_FILE: ${{ steps.determine_prompt.outputs.PROMPT_FILE_PATH }}
+        run: |
+          echo "Running probe-chat to find relevant files..."
+          if [[ ! -f "$PROMPT_CONTENT_FILE" ]]; then
+            echo "::error::Prompt file $PROMPT_CONTENT_FILE not found!"
+            exit 1
+          fi
+          PROMPT_CONTENT=$(cat "$PROMPT_CONTENT_FILE")
+          if [ -z "$PROMPT_CONTENT" ]; then
+             echo "::error::Prompt content is empty!"
+             exit 1
+          fi
+
+          PROMPT_ESCAPED=$(jq -Rs . <<< "$PROMPT_CONTENT")
+
+          MESSAGE_FOR_PROBE=$(jq -n --arg prompt_escaped "$PROMPT_ESCAPED" \
+            '{ "message": "I'\''m giving you a request that needs to be implemented. Your role is ONLY to give me the files that are relevant to the request and nothing else. The request is prepended with the word REQUEST.\\nREQUEST: \($prompt_escaped). Give me all the files relevant to this request. Your output MUST be a single json array that can be parsed with programatic json parsing, with the relevant files. Files can be rust or typescript or javascript files. DO NOT INCLUDE ANY OTHER TEXT IN YOUR OUTPUT. ONLY THE JSON ARRAY. Example of output: [\"file1.py\", \"file2.py\"]" }' | jq -r .message)
+
+          set -o pipefail
+          PROBE_OUTPUT=$(npx --yes @buger/probe-chat@latest --max-iterations 50 --model-name gemini-2.5-pro-preview-05-06 --message "$MESSAGE_FOR_PROBE") || {
+            echo "::error::probe-chat command failed. Output:"
+            echo "$PROBE_OUTPUT"
+            exit 1
+          }
+          set +o pipefail
+          echo "Probe-chat raw output:"
+          echo "$PROBE_OUTPUT"
+
+          JSON_FILES=$(echo "$PROBE_OUTPUT" | sed -n '/^\s*\[/,$p' | sed '/^\s*\]/q')
+          echo "Extracted JSON block:"
+          echo "$JSON_FILES"
+
+          FILES_LIST=$(echo "$JSON_FILES" | jq -e -r '[.[] | select(type == "string" and . != "" and . != null and (endswith("/") | not))] | map(@sh) | join(" ")' || echo "")
+
+          if [[ -z "$FILES_LIST" ]]; then
+             echo "::warning::probe-chat did not identify any relevant files."
+             exit 1
+          fi
+
+          echo "Formatted files list for aider: $FILES_LIST"
+          echo "FILES_TO_EDIT=$FILES_LIST" >> $GITHUB_ENV
+
+      - name: Run Aider with external prompt
+        run: |
+          echo "Files identified by probe-chat: ${{ env.FILES_TO_EDIT }}"
+          aider \
+            --read .cursor/rules/rust-best-practices.mdc \
+            --read .cursor/rules/svelte5-best-practices.mdc \
+            ${{ env.FILES_TO_EDIT }} \
+            --model gemini/gemini-2.5-pro-preview-05-06 \
+            --message-file .github/aider/issue-prompt.txt \
+            --yes \
+            --no-check-update \
+            --auto-commits \
+            --no-analytics \
+            --no-gitignore \
+            | tee .github/aider/aider-output.txt || true
+          echo "Aider command completed. Output saved to .github/aider/aider-output.txt"
+
+      - name: Clean up prompt file
+        if: always()
+        run: rm -f .github/aider/issue-prompt.txt
+
+      - name: Commit and Push Changes
+        id: commit_and_push
+        if: ${{ success() }}
+        run: |
+          if [[ -z "${{ github.event.issue.pull_request }}" ]]; then
+            BRANCH_NAME="aider-fix-issue-${{ github.event.issue.number }}"
+            
+            # Check if branch exists remotely
+            if git ls-remote --heads origin $BRANCH_NAME | grep -q $BRANCH_NAME; then
+              echo "Branch $BRANCH_NAME already exists remotely, fetching it"
+              git fetch origin $BRANCH_NAME
+              git checkout $BRANCH_NAME
+              git pull origin $BRANCH_NAME
+            else
+              echo "Creating new branch $BRANCH_NAME"
+              git checkout -b $BRANCH_NAME
+            fi
+            
+            echo "Created/checked out branch $BRANCH_NAME for issue #${{ github.event.issue.number }}"
+            git push origin $BRANCH_NAME
+            echo "Pushed to branch $BRANCH_NAME"
+            echo "PR_BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT
+            echo "CHANGES_APPLIED_MESSAGE=Aider changes pushed to branch $BRANCH_NAME." >> $GITHUB_OUTPUT
+          else
+            CURRENT_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD)
+            echo "Attempting to push changes to PR branch $CURRENT_BRANCH_NAME for PR #${{ github.event.issue.number }}"
+            if git push origin $CURRENT_BRANCH_NAME; then
+              echo "Push to $CURRENT_BRANCH_NAME successful (or no new changes to push)."
+              echo "CHANGES_APPLIED_MESSAGE=Aider changes (if any) pushed to PR branch $CURRENT_BRANCH_NAME." >> $GITHUB_OUTPUT
+              echo "PR_BRANCH_NAME=$CURRENT_BRANCH_NAME" >> $GITHUB_OUTPUT
+            else
+              echo "::warning::Push to PR branch $CURRENT_BRANCH_NAME failed."
+              echo "CHANGES_APPLIED_MESSAGE=Aider ran, but failed to push changes to PR branch $CURRENT_BRANCH_NAME." >> $GITHUB_OUTPUT
+            fi
+          fi
+
+      - name: Create Pull Request
+        if: success() && github.event_name == 'issue_comment' && !github.event.issue.pull_request && steps.commit_and_push.outputs.PR_BRANCH_NAME != ''
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_BRANCH: ${{ steps.commit_and_push.outputs.PR_BRANCH_NAME }}
+          ISSUE_NUM: ${{ github.event.issue.number }}
+        run: |
+          # Create PR description in a temporary file to avoid command line length limits
+          cat > /tmp/pr-description.md << EOL
+          This PR was created automatically by Aider to fix issue #${ISSUE_NUM}.
+
+          ## Aider Output
+          \`\`\`
+          $(cat .github/aider/aider-output.txt || echo "No output available")
+          \`\`\`
+          EOL
+
+          # Create PR using the file for the body content
+          gh pr create \
+            --title "[Aider PR] Add fixes for issue #${ISSUE_NUM}" \
+            --body-file /tmp/pr-description.md \
+            --head "$PR_BRANCH" \
+            --base main

.github/workflows/create-docs.yml

@@ -0,0 +1,15 @@
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  trigger-docs:
+    if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/docs') }}
+    uses: windmill-labs/windmilldocs/.github/workflows/create-docs.yml@main
+    with:
+      pr_number: ${{ github.event.issue.number }}
+      repo: ${{ github.event.repository.name }}
+      comment_text: ${{ github.event.comment.body }}
+    secrets:
+      DOCS_TOKEN: ${{ secrets.DOCS_TOKEN }}
+      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}

.github/workflows/discord-notification.yml

@@ -0,0 +1,32 @@
+name: Create discord thread when a PR is opened, react with green checkmark when PR is merged
+
+on:
+  pull_request:
+    types:
+      - opened
+      - ready_for_review
+      - closed
+
+jobs:
+  notify_discord_when_pr_opened:
+    if: (github.event.pull_request.draft == false) && (github.event.action == 'opened' || github.event.action == 'ready_for_review')
+    uses: ./.github/workflows/shareable-discord-notification.yml
+    with:
+      PR_TITLE: ${{ github.event.pull_request.title }}
+      PR_URL: ${{ github.event.pull_request.html_url }}
+      PR_AUTHOR: ${{ github.event.pull_request.user.login }}
+      PR_STATUS: "opened"
+      PR_NUMBER: ${{ github.event.pull_request.number }}
+    secrets:
+      DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_PR_REVIEWS_WEBHOOK }}
+
+  merge_success_emoji:
+    if: github.event.pull_request.merged == true
+    uses: ./.github/workflows/shareable-discord-notification.yml
+    with:
+      PR_STATUS: "merged"
+      DISCORD_CHANNEL_ID: "1372204995868491786"
+      DISCORD_GUILD_ID: "930051556043276338"
+      PR_NUMBER: ${{ github.event.pull_request.number }}
+    secrets:
+      DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_PR_BOT_TOKEN }}

.github/workflows/helmchart_on_release.yml

@@ -0,0 +1,66 @@
+name: Publish Helm Chart on Release
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  bump-helm-version:
+    runs-on: ubicloud-standard-2
+
+    steps:
+      - name: Checkout on helm repository
+        uses: actions/checkout@v3
+        with:
+          repository: windmill-labs/windmill-helm-charts
+          token: ${{ secrets.DOCS_TOKEN }}
+
+      - name: Get version
+        id: get_version
+        run: |
+          echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
+
+      - name: Create new branch
+        run: |
+          # Check if branch already exists remotely
+          if git ls-remote --heads origin bump-helm-version-${{ env.VERSION }} | grep -q bump-helm-version-${{ env.VERSION }}; then
+            # Branch exists, check it out
+            git fetch origin bump-helm-version-${{ env.VERSION }}
+            git checkout bump-helm-version-${{ env.VERSION }}
+          else
+            # Create new branch
+            git checkout -b bump-helm-version-${{ env.VERSION }}
+          fi
+
+          git config --local user.email "action@github.com"
+          git config --local user.name "GitHub Action"
+
+      - name: Bump helm version
+        run: |
+          # Get current version and increment it by 1
+          CURRENT_VERSION=$(grep "version:" ./charts/windmill/Chart.yaml | awk '{print $2}' | head -n 1)
+          NEW_VERSION=$(echo "$CURRENT_VERSION" | awk -F. '{$NF = $NF + 1;} 1' | sed 's/ /./g')
+          sed -i "s/^version: .*/version: $NEW_VERSION/" ./charts/windmill/Chart.yaml
+
+          # Get the app version from the version
+          VERSION=${{ env.VERSION }}
+          APP_VERSION=${VERSION#refs/tag/}
+          APP_VERSION=${APP_VERSION#v}
+          APP_VERSION=${APP_VERSION%/}
+          sed -i "s/appVersion: .*/appVersion: $APP_VERSION/" ./charts/windmill/Chart.yaml
+
+      - name: Commit and push
+        run: |
+          git add .
+          git commit -m "Bump helm version to ${{ env.VERSION }}"
+          git push origin bump-helm-version-${{ env.VERSION }}
+
+      - name: Create PR
+        env:
+          GH_TOKEN: ${{ secrets.DOCS_TOKEN }}
+        run: |
+          gh pr create \
+            --title "helm: bump version to ${{ env.VERSION }}" \
+            --body "This PR was auto-generated to bring the helm chart up to date for [release ${{ env.VERSION }}](https://github.com/windmill-labs/windmill/releases/tag/v${{ env.VERSION }}) in the main repo." \
+            --head bump-helm-version-${{ env.VERSION }} \
+            --base main

.github/workflows/shareable-discord-notification.yml

@@ -0,0 +1,98 @@
+name: "Notify Discord when a PR is opened or merged"
+
+on:
+  workflow_call:
+    inputs:
+      PR_TITLE:
+        description: "The title of the PR"
+        type: string
+      PR_URL:
+        description: "The URL of the PR"
+        type: string
+      PR_AUTHOR:
+        description: "The author of the PR"
+        type: string
+      PR_STATUS:
+        description: "The status of the PR"
+        type: string
+      DISCORD_CHANNEL_ID:
+        description: "The Discord channel ID"
+        type: string
+      PR_NUMBER:
+        description: "The number of the PR"
+        type: string
+      DISCORD_GUILD_ID:
+        description: "The Discord guild ID"
+        type: string
+    secrets:
+      DISCORD_WEBHOOK_URL:
+        description: "Discord Webhook URL"
+      DISCORD_BOT_TOKEN:
+        description: "Discord Bot Token"
+
+jobs:
+  open_thread:
+    runs-on: ubicloud-standard-2
+    if: ${{ inputs.PR_STATUS == 'opened' }}
+    steps:
+      - name: Send Discord notification and start a thread
+        env:
+          WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
+          PR_TITLE: ${{ inputs.PR_TITLE }}
+          PR_NUMBER: ${{ inputs.PR_NUMBER }}
+          PR_URL: ${{ inputs.PR_URL }}
+          PR_AUTHOR: ${{ inputs.PR_AUTHOR }}
+        run: |
+          payload=$(jq -n \
+            --arg content "${PR_URL}" \
+            --arg thread "#${PR_NUMBER}: $PR_TITLE by \`${PR_AUTHOR}\`" \
+            '{
+              content: $content,
+              thread_name: $thread,
+              auto_archive_duration: 10080
+            }'
+          )
+          curl -H "Content-Type: application/json" \
+            -X POST \
+            -d "$payload" \
+            "$WEBHOOK_URL"
+
+  merge_success_emoji:
+    runs-on: ubuntu-latest
+    if: ${{ inputs.PR_STATUS == 'merged' }}
+    steps:
+      - name: React
+        env:
+          BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }}
+          CHANNEL_ID: ${{ inputs.DISCORD_CHANNEL_ID }}
+          GUILD_ID: ${{ inputs.DISCORD_GUILD_ID }}
+          PR_NUMBER: ${{ inputs.PR_NUMBER }}
+        run: |
+          # 1) get PR thread
+          threads=$(curl -H "Authorization: Bot $BOT_TOKEN" "https://discord.com/api/v10/guilds/${GUILD_ID}/threads/active")
+          thread_id=$(
+            echo "$threads" \
+              | jq -r --arg cid "$CHANNEL_ID" \
+                    --arg pref "#${PR_NUMBER}:" \
+                '.threads[]
+                | select(.parent_id == $cid and (.name | startswith($pref)))
+                | .id'
+          )
+          if [ -z "$thread_id" ]; then
+            echo "Thread not found"
+            exit 1
+          fi
+          # 2) get the first message in that thread
+          messages=$(curl -H "Authorization: Bot $BOT_TOKEN" \
+            "https://discord.com/api/v10/channels/$thread_id/messages?limit=1")
+          message_id=$(echo "$messages" | jq -r '.[-1].id')
+
+          if [ -z "$message_id" ]; then
+            echo "Message not found"
+            exit 1
+          fi
+
+          # 3) add the ✅ reaction
+          curl -X PUT \
+            -H "Authorization: Bot $BOT_TOKEN" \
+            "https://discord.com/api/v10/channels/$thread_id/messages/$message_id/reactions/%E2%9C%85/@me"

.gitignore

@@ -10,3 +10,5 @@ CaddyfileRemoteMalo
 .vscode
 .dev-docker-wrapper*
 backend/.minio-data
+.aider*
+!.aiderignore

CHANGELOG.md

@@ -1,5 +1,66 @@
 # Changelog
 
+## [1.491.5](https://github.com/windmill-labs/windmill/compare/v1.491.4...v1.491.5) (2025-05-17)
+
+
+### Bug Fixes
+
+* improve handling of custom concurrency key/tag with preprocessors ([#5762](https://github.com/windmill-labs/windmill/issues/5762)) ([59afa49](https://github.com/windmill-labs/windmill/commit/59afa493fa20cc70b6825e6356713cef84d75312))
+* S3 sql mode returns S3Object ([#5764](https://github.com/windmill-labs/windmill/issues/5764)) ([b29c6e7](https://github.com/windmill-labs/windmill/commit/b29c6e7636bb21c4d977bdaf89ac90e2a1a1086c))
+
+## [1.491.4](https://github.com/windmill-labs/windmill/compare/v1.491.3...v1.491.4) (2025-05-15)
+
+
+### Bug Fixes
+
+* add v1 preprocessor support to workspace preprocessor script ([#5757](https://github.com/windmill-labs/windmill/issues/5757)) ([9b1c30e](https://github.com/windmill-labs/windmill/commit/9b1c30eeff35291ad50f3ddeb64831eac88e2f66))
+
+## [1.491.3](https://github.com/windmill-labs/windmill/compare/v1.491.2...v1.491.3) (2025-05-15)
+
+
+### Bug Fixes
+
+* **frontend:** fix accordeon tabs initialization ([f488903](https://github.com/windmill-labs/windmill/commit/f488903635a1457f839ca641ed4f8d0891ef8212))
+* http trigger routers cache version sequence ([#5755](https://github.com/windmill-labs/windmill/issues/5755)) ([d53bceb](https://github.com/windmill-labs/windmill/commit/d53bceb8004541b79d33220ae8de06d25521da91))
+
+## [1.491.2](https://github.com/windmill-labs/windmill/compare/v1.491.1...v1.491.2) (2025-05-15)
+
+
+### Bug Fixes
+
+* **cli:** --version improvement ([f8f2015](https://github.com/windmill-labs/windmill/commit/f8f201564f7a323eb96f6dc684a525a0784d41f2))
+* http trigger signature validation ([#5753](https://github.com/windmill-labs/windmill/issues/5753)) ([9e9514b](https://github.com/windmill-labs/windmill/commit/9e9514b9af2337e143a9e4cf1e915e1477032e80))
+* Improve indexer performance by factoring required queries to the DB # ([#5749](https://github.com/windmill-labs/windmill/issues/5749)) ([b12feaf](https://github.com/windmill-labs/windmill/commit/b12feaf50ae0ef03816719ff39157fcf55159dbf))
+* improve perf of job deletion ([0efba94](https://github.com/windmill-labs/windmill/commit/0efba945bac9b84a489c6ef552e834593f209fe1))
+
+
+### Performance Improvements
+
+* cache http trigger routers and auth ([#5748](https://github.com/windmill-labs/windmill/issues/5748)) ([ddd18d2](https://github.com/windmill-labs/windmill/commit/ddd18d22a615408a9f57f910d0a58f17e6d6e29d))
+
+## [1.491.1](https://github.com/windmill-labs/windmill/compare/v1.491.0...v1.491.1) (2025-05-15)
+
+
+### Bug Fixes
+
+* avoid deadlocks in sending completed job to result processors ([#5742](https://github.com/windmill-labs/windmill/issues/5742)) ([e87d4f3](https://github.com/windmill-labs/windmill/commit/e87d4f3c1afb4ad356b326b7600c89e6c7803eff))
+
+## [1.491.0](https://github.com/windmill-labs/windmill/compare/v1.490.0...v1.491.0) (2025-05-14)
+
+
+### Features
+
+* Microsoft Teams approvals ([#5734](https://github.com/windmill-labs/windmill/issues/5734)) ([039f3e0](https://github.com/windmill-labs/windmill/commit/039f3e02268f2acda48abea420479216970e58e7))
+* sql jobs outputting to s3 + streaming for high-number of rows ([#5704](https://github.com/windmill-labs/windmill/issues/5704)) ([c7886ea](https://github.com/windmill-labs/windmill/commit/c7886ea07ae44af56f1467288b2d73ff2ae27964))
+
+
+### Bug Fixes
+
+* add missing run job transaction drop ([#5730](https://github.com/windmill-labs/windmill/issues/5730)) ([318def9](https://github.com/windmill-labs/windmill/commit/318def976cf0e4d5c32d01ac611a89e0a6425368))
+* add support for log compaction on docker jobs ([#5732](https://github.com/windmill-labs/windmill/issues/5732)) ([d35a7d2](https://github.com/windmill-labs/windmill/commit/d35a7d22f960f485889e22de48e8de8557069cb7))
+* Ansible lockfile back compatibility issue ([#5731](https://github.com/windmill-labs/windmill/issues/5731)) ([f73c90c](https://github.com/windmill-labs/windmill/commit/f73c90c7518569204b298b916d0fc298932d3cf0))
+* trigger event support for webhook get endpoints ([#5728](https://github.com/windmill-labs/windmill/issues/5728)) ([76258b7](https://github.com/windmill-labs/windmill/commit/76258b7b1af1313f694731d77f3fa6994e9ded70))
+
 ## [1.490.0](https://github.com/windmill-labs/windmill/compare/v1.489.0...v1.490.0) (2025-05-12)
 
 

backend/.sqlx/query-0e296134f05593edc989c628c00cbb60a5446993217baffa83f843bc12a5ac73.json

@@ -1,35 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT\n                    value->'preprocessor_module' IS NOT NULL as has_preprocessor,\n                    value->'preprocessor_module'->'value'->'input_transforms'->'wm_trigger' IS NOT NULL as is_v1_preprocessor,\n                    schema as \"schema: _\"\n                FROM flow \n                WHERE workspace_id = $1 \n                    AND path = $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "has_preprocessor",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 1,
-        "name": "is_v1_preprocessor",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 2,
-        "name": "schema: _",
-        "type_info": "Json"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      null,
-      null,
-      true
-    ]
-  },
-  "hash": "0e296134f05593edc989c628c00cbb60a5446993217baffa83f843bc12a5ac73"
-}

backend/.sqlx/query-16be720bf1c88ecfa2a4bf6adbb1924df4817b6236b3a949209465f3a2c42bb9.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT nextval('http_trigger_version_seq')",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "nextval",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "16be720bf1c88ecfa2a4bf6adbb1924df4817b6236b3a949209465f3a2c42bb9"
+}

backend/.sqlx/query-1d819b829cd92995c39d29540df8cffbcc3334bada244a331a0bd8db06029d42.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM v2_job_completed c\n                    USING v2_job j\n                    WHERE\n                        created_at <= now() - ($1::bigint::text || ' s')::interval\n                        AND completed_at + ($1::bigint::text || ' s')::interval <= now()\n                        AND c.id = j.id\n                    RETURNING c.id",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Uuid"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int8"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "1d819b829cd92995c39d29540df8cffbcc3334bada244a331a0bd8db06029d42"
-}

backend/.sqlx/query-1eeb218c30c0a6b0f7633813c764f57f8968894b4786b94109a057796ff500e4.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n            SELECT \n                path, \n                script_path, \n                is_flow, \n                route_path, \n                authentication_resource_path,\n                workspace_id, \n                is_async, \n                authentication_method  AS \"authentication_method: _\", \n                edited_by, \n                email, \n                static_asset_config AS \"static_asset_config: _\",\n                wrap_body,\n                raw_string,\n                workspaced_route,\n                is_static_website\n            FROM \n                http_trigger \n            WHERE \n                http_method = $1\n            ",
+  "query": "\n                    SELECT \n                        path, \n                        script_path, \n                        is_flow, \n                        route_path, \n                        authentication_resource_path,\n                        workspace_id, \n                        is_async, \n                        authentication_method  AS \"authentication_method: _\", \n                        edited_by, \n                        email, \n                        static_asset_config AS \"static_asset_config: _\",\n                        wrap_body,\n                        raw_string,\n                        workspaced_route,\n                        is_static_website\n                    FROM \n                        http_trigger \n                    WHERE \n                        http_method = $1\n                    ",
   "describe": {
     "columns": [
       {
@@ -129,5 +129,5 @@
       false
     ]
   },
-  "hash": "4053f0bb30f651ddf2214115748daca0ea457da8252394eeeead0897d184f6da"
+  "hash": "1eeb218c30c0a6b0f7633813c764f57f8968894b4786b94109a057796ff500e4"
 }

backend/.sqlx/query-20fd50949796913dd48f67ee75b11272ce5b3046f87b9efd504e013fba9724f5.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT last_value FROM http_trigger_version_seq",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "last_value",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "20fd50949796913dd48f67ee75b11272ce5b3046f87b9efd504e013fba9724f5"
+}

backend/.sqlx/query-234acda79d470e99e9cbde5c7401d6f7894c25f90e39ec8606f79b8be56d1c17.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                        SELECT EXISTS(\n                            SELECT 1\n                            FROM \n                                http_trigger \n                            WHERE \n                                workspace_id = $1 AND \n                                path = $2\n                        )\n                        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "234acda79d470e99e9cbde5c7401d6f7894c25f90e39ec8606f79b8be56d1c17"
+}

backend/.sqlx/query-282afbff89d3186d47ef5dbd0b65026ad37fb31b485fc44b6ec257dd77825428.json

@@ -1,41 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT created_by, coalesce(job_logs.logs, '') as logs, job_logs.log_offset, job_logs.log_file_index\n        FROM v2_as_completed_job\n        LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id\n        WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "created_by",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 1,
-        "name": "logs",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 2,
-        "name": "log_offset",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 3,
-        "name": "log_file_index",
-        "type_info": "TextArray"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Uuid",
-        "Text"
-      ]
-    },
-    "nullable": [
-      true,
-      null,
-      false,
-      true
-    ]
-  },
-  "hash": "282afbff89d3186d47ef5dbd0b65026ad37fb31b485fc44b6ec257dd77825428"
-}

backend/.sqlx/query-2a33a35afc1ba4c31a5713cfd1a2c662f25cda387197aaf9f35000df31b8b07d.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DROP INDEX CONCURRENTLY IF EXISTS log_file_hostname_log_ts_idx",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "2a33a35afc1ba4c31a5713cfd1a2c662f25cda387197aaf9f35000df31b8b07d"
+}

backend/.sqlx/query-3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e.json

@@ -0,0 +1,91 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "WITH job_info AS (\n            -- Query for Teams (running jobs)\n            SELECT\n                parent.job_kind AS \"job_kind!: JobKind\",\n                parent.script_hash AS \"script_hash: ScriptHash\",\n                parent.raw_flow AS \"raw_flow: sqlx::types::Json<Box<RawValue>>\",\n                child.parent_job AS \"parent_job: Uuid\",\n                parent.created_at AS \"created_at!: chrono::NaiveDateTime\",\n                parent.created_by AS \"created_by!\",\n                parent.script_path,\n                parent.args AS \"args: sqlx::types::Json<Box<RawValue>>\"\n            FROM v2_as_queue child\n            JOIN v2_as_queue parent ON parent.id = child.parent_job\n            WHERE child.id = $1 AND child.workspace_id = $2\n            UNION ALL\n            -- Query for Slack (completed jobs)\n            SELECT\n                v2_as_queue.job_kind AS \"job_kind!: JobKind\",\n                v2_as_queue.script_hash AS \"script_hash: ScriptHash\",\n                v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json<Box<RawValue>>\",\n                v2_as_completed_job.parent_job AS \"parent_job: Uuid\",\n                v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\",\n                v2_as_completed_job.created_by AS \"created_by!\",\n                v2_as_queue.script_path,\n                v2_as_queue.args AS \"args: sqlx::types::Json<Box<RawValue>>\"\n            FROM v2_as_queue\n            JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id\n            WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2\n        )\n        SELECT * FROM job_info LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "job_kind!: JobKind",
+        "type_info": {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies",
+                "deploymentcallback",
+                "singlescriptflow",
+                "flowscript",
+                "flownode",
+                "appscript"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 1,
+        "name": "script_hash: ScriptHash",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "raw_flow: sqlx::types::Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 3,
+        "name": "parent_job: Uuid",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 4,
+        "name": "created_at!: chrono::NaiveDateTime",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 5,
+        "name": "created_by!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "script_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 7,
+        "name": "args: sqlx::types::Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null
+    ]
+  },
+  "hash": "3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e"
+}

backend/.sqlx/query-49943f69ed74bc889120dcd2571e8e868a4f4795933044ff95b20d8df45cd145.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_job_completed_completed_at ON v2_job_completed (completed_at DESC)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "49943f69ed74bc889120dcd2571e8e868a4f4795933044ff95b20d8df45cd145"
+}

backend/.sqlx/query-52ad0c838d19cbd9e90b8368abe71dd12655179f41f43896e7d30fdfb3ae5939.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "WITH job_result AS (\n                 SELECT result \n                 FROM v2_job_completed \n                 WHERE id = $1\n             )\n             UPDATE v2_job \n             SET args = COALESCE(\n                     CASE \n                         WHEN job_result.result IS NULL THEN NULL\n                         WHEN jsonb_typeof(job_result.result) = 'object' \n                         THEN job_result.result\n                         WHEN jsonb_typeof(job_result.result) = 'null'\n                         THEN NULL\n                         ELSE jsonb_build_object('value', job_result.result)\n                     END, \n                     '{}'::jsonb\n                 ),\n                 preprocessed = TRUE\n             FROM job_result\n             WHERE v2_job.id = $2;\n             ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Uuid",
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "52ad0c838d19cbd9e90b8368abe71dd12655179f41f43896e7d30fdfb3ae5939"
-}

backend/.sqlx/query-5820d34be1a7f7b72e656c692f53146f45ad4a6e584e917a0a86280d8f473c10.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM v2_job_completed c\n                    WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval \n                    RETURNING c.id",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "5820d34be1a7f7b72e656c692f53146f45ad4a6e584e917a0a86280d8f473c10"
+}

backend/.sqlx/query-6e2564dc37ee967c634deade67ceabc4c516418e595dee9cd7d651acc1f4af6e.json

@@ -0,0 +1,248 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n                            cj.id AS \"id!\",\n                            cj.workspace_id AS \"workspace_id!\",\n                            cj.parent_job,\n                            cj.created_by AS \"created_by!\",\n                            cj.duration_ms AS \"duration_ms!\",\n                            cj.success AS \"success!\",\n                            cj.script_hash AS \"script_hash!: Option<ScriptHash>\",\n                            cj.script_path,\n                            cj.args AS \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\",\n                            cj.result AS \"result: sqlx::types::Json<Box<RawValue>>\",\n                            cj.deleted AS \"deleted!\",\n                            cj.canceled AS \"canceled!\",\n                            cj.canceled_by,\n                            cj.canceled_reason,\n                            cj.job_kind AS \"job_kind!: JobKind\",\n                            cj.schedule_path,\n                            cj.permissioned_as AS \"permissioned_as!\",\n                            cj.is_flow_step AS \"is_flow_step!\",\n                            cj.language AS \"language: ScriptLang\",\n                            cj.is_skipped AS \"is_skipped!\",\n                            cj.email AS \"email!\",\n                            cj.visible_to_owner AS \"visible_to_owner!\",\n                            cj.mem_peak,\n                            cj.tag AS \"tag!\",\n                            cj.created_at AS \"created_at!\",\n                            cj.started_at,\n                            job_logs.logs,\n                            job_logs.log_offset AS \"log_offset?\",\n                            job_logs.log_file_index\n\n                        FROM v2_as_completed_job AS cj\n                        LEFT JOIN job_logs ON cj.id = job_logs.job_id\n                        WHERE (cj.created_at > $1 AND cj.created_at < $3)\n                               OR cj.id = ANY($2)\n                        ORDER BY cj.created_at ASC LIMIT $4",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id!",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 1,
+        "name": "workspace_id!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "parent_job",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 3,
+        "name": "created_by!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 4,
+        "name": "duration_ms!",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 5,
+        "name": "success!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 6,
+        "name": "script_hash!: Option<ScriptHash>",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 7,
+        "name": "script_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 8,
+        "name": "args: sqlx::types::Json<HashMap<String, Box<RawValue>>>",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 9,
+        "name": "result: sqlx::types::Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 10,
+        "name": "deleted!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 11,
+        "name": "canceled!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 12,
+        "name": "canceled_by",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 13,
+        "name": "canceled_reason",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 14,
+        "name": "job_kind!: JobKind",
+        "type_info": {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies",
+                "deploymentcallback",
+                "singlescriptflow",
+                "flowscript",
+                "flownode",
+                "appscript"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 15,
+        "name": "schedule_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 16,
+        "name": "permissioned_as!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 17,
+        "name": "is_flow_step!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 18,
+        "name": "language: ScriptLang",
+        "type_info": {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell",
+                "mssql",
+                "php",
+                "bunnative",
+                "rust",
+                "ansible",
+                "csharp",
+                "oracledb",
+                "nu",
+                "java"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 19,
+        "name": "is_skipped!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 20,
+        "name": "email!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 21,
+        "name": "visible_to_owner!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 22,
+        "name": "mem_peak",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 23,
+        "name": "tag!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 24,
+        "name": "created_at!",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 25,
+        "name": "started_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 26,
+        "name": "logs",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 27,
+        "name": "log_offset?",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 28,
+        "name": "log_file_index",
+        "type_info": "TextArray"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Timestamptz",
+        "UuidArray",
+        "Timestamptz",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      false,
+      true
+    ]
+  },
+  "hash": "6e2564dc37ee967c634deade67ceabc4c516418e595dee9cd7d651acc1f4af6e"
+}

backend/.sqlx/query-72916f8e490f8252e0a51b7f562ccc3be832b12102eb86a07d8405a4fa9287d5.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n                    value->'preprocessor_module'->'value' as \"preprocessor_module: _\",\n                    schema as \"schema: _\"\n                FROM flow \n                WHERE workspace_id = $1\n                    AND path = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "preprocessor_module: _",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 1,
+        "name": "schema: _",
+        "type_info": "Json"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null,
+      true
+    ]
+  },
+  "hash": "72916f8e490f8252e0a51b7f562ccc3be832b12102eb86a07d8405a4fa9287d5"
+}

backend/.sqlx/query-91f23fcc27777c279c79e2682fc15c026e55f9ec3799be65a2e8920fe6174a17.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT result  as \"result: Json<HashMap<String, Box<RawValue>>>\"\n                 FROM v2_job_completed \n                 WHERE id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "result: Json<HashMap<String, Box<RawValue>>>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "91f23fcc27777c279c79e2682fc15c026e55f9ec3799be65a2e8920fe6174a17"
+}

backend/.sqlx/query-927149213e0f8ae983652ef80464f646d6be80e702193f1acdd40dd6033652e4.json

@@ -1,134 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            SELECT \n                path, \n                script_path, \n                is_flow, \n                route_path, \n                workspace_id, \n                is_async, \n                authentication_method  AS \"authentication_method: _\", \n                edited_by, \n                email,\n                static_asset_config AS \"static_asset_config: _\",\n                wrap_body,\n                raw_string,\n                workspaced_route,\n                is_static_website,\n                authentication_resource_path\n            FROM \n                http_trigger \n            WHERE \n                workspace_id = $1 AND \n                http_method = $2\n            ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "path",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 1,
-        "name": "script_path",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 2,
-        "name": "is_flow",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 3,
-        "name": "route_path",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 4,
-        "name": "workspace_id",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 5,
-        "name": "is_async",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 6,
-        "name": "authentication_method: _",
-        "type_info": {
-          "Custom": {
-            "name": "authentication_method",
-            "kind": {
-              "Enum": [
-                "none",
-                "windmill",
-                "api_key",
-                "basic_http",
-                "custom_script",
-                "signature"
-              ]
-            }
-          }
-        }
-      },
-      {
-        "ordinal": 7,
-        "name": "edited_by",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 8,
-        "name": "email",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 9,
-        "name": "static_asset_config: _",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 10,
-        "name": "wrap_body",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 11,
-        "name": "raw_string",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 12,
-        "name": "workspaced_route",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 13,
-        "name": "is_static_website",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 14,
-        "name": "authentication_resource_path",
-        "type_info": "Varchar"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        {
-          "Custom": {
-            "name": "http_method",
-            "kind": {
-              "Enum": [
-                "get",
-                "post",
-                "put",
-                "delete",
-                "patch"
-              ]
-            }
-          }
-        }
-      ]
-    },
-    "nullable": [
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      true,
-      false,
-      false,
-      false,
-      false,
-      true
-    ]
-  },
-  "hash": "927149213e0f8ae983652ef80464f646d6be80e702193f1acdd40dd6033652e4"
-}

backend/.sqlx/query-a8ca4e588e0bf3c4bba2fe4b68a5364e4cba99964513599f8a012a5680d3dca8.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS alerts_by_workspace ON alerts (workspace_id);",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "a8ca4e588e0bf3c4bba2fe4b68a5364e4cba99964513599f8a012a5680d3dca8"
+}

backend/.sqlx/query-b7335ac24702c86fbb4ab95916a6aa1648082287b09122755df2462dc71ce831.json

@@ -0,0 +1,46 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        WITH j AS (\n            SELECT \n                raw_flow->>'concurrency_key' as concurrency_key, \n                raw_flow->>'concurrency_time_window_s' as concurrency_time_window_s,\n                raw_flow->>'concurrency_limit' as concurrent_limit,\n                runnable_path, \n                runnable_id as version FROM v2_job\n            WHERE id = $1\n        )\n        SELECT tag, j.concurrency_key, j.concurrency_time_window_s::int, j.concurrent_limit::int, j.version\n            FROM flow, j\n            WHERE path = j.runnable_path\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "tag",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "concurrency_key",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "concurrency_time_window_s",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 3,
+        "name": "concurrent_limit",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 4,
+        "name": "version",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      true,
+      null,
+      null,
+      null,
+      true
+    ]
+  },
+  "hash": "b7335ac24702c86fbb4ab95916a6aa1648082287b09122755df2462dc71ce831"
+}

backend/.sqlx/query-e07660e8d2a265cb6a83f3a2bb8e7e6330f09ab116e9837f6f16f8fdef938004.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "WITH job_result AS (\n                 SELECT result \n                 FROM v2_job_completed \n                 WHERE id = $1\n             ),\n             updated_queue AS (\n                UPDATE v2_job_queue\n                SET running = false,\n                tag = COALESCE($3, tag)\n                WHERE id = $2\n             )\n             UPDATE v2_job \n             SET \n                tag = COALESCE($3, tag),\n                concurrent_limit = COALESCE($4, concurrent_limit),\n                concurrency_time_window_s = COALESCE($5, concurrency_time_window_s),\n                args = COALESCE(\n                     CASE \n                         WHEN job_result.result IS NULL THEN NULL\n                         WHEN jsonb_typeof(job_result.result) = 'object' \n                         THEN job_result.result\n                         WHEN jsonb_typeof(job_result.result) = 'null'\n                         THEN NULL\n                         ELSE jsonb_build_object('value', job_result.result)\n                     END, \n                     '{}'::jsonb\n                 ),\n                 preprocessed = TRUE\n             FROM job_result\n             WHERE v2_job.id = $2;\n             ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Uuid",
+        "Varchar",
+        "Int4",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "e07660e8d2a265cb6a83f3a2bb8e7e6330f09ab116e9837f6f16f8fdef938004"
+}

backend/.sqlx/query-f0fdeb7aea3e71099e7db0f4343bbd7ec86610ddc8589bf5b606fab0947c8b75.json

@@ -1,91 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT\n            v2_as_queue.job_kind AS \"job_kind!: JobKind\",\n            v2_as_queue.script_hash AS \"script_hash: ScriptHash\",\n            v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json<Box<RawValue>>\",\n            v2_as_completed_job.parent_job AS \"parent_job: Uuid\",\n            v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\",\n            v2_as_completed_job.created_by AS \"created_by!\",\n            v2_as_queue.script_path,\n            v2_as_queue.args AS \"args: sqlx::types::Json<Box<RawValue>>\"\n        FROM v2_as_queue\n        JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id\n        WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2\n        LIMIT 1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "job_kind!: JobKind",
-        "type_info": {
-          "Custom": {
-            "name": "job_kind",
-            "kind": {
-              "Enum": [
-                "script",
-                "preview",
-                "flow",
-                "dependencies",
-                "flowpreview",
-                "script_hub",
-                "identity",
-                "flowdependencies",
-                "http",
-                "graphql",
-                "postgresql",
-                "noop",
-                "appdependencies",
-                "deploymentcallback",
-                "singlescriptflow",
-                "flowscript",
-                "flownode",
-                "appscript"
-              ]
-            }
-          }
-        }
-      },
-      {
-        "ordinal": 1,
-        "name": "script_hash: ScriptHash",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 2,
-        "name": "raw_flow: sqlx::types::Json<Box<RawValue>>",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 3,
-        "name": "parent_job: Uuid",
-        "type_info": "Uuid"
-      },
-      {
-        "ordinal": 4,
-        "name": "created_at!: chrono::NaiveDateTime",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 5,
-        "name": "created_by!",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 6,
-        "name": "script_path",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 7,
-        "name": "args: sqlx::types::Json<Box<RawValue>>",
-        "type_info": "Jsonb"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Uuid",
-        "Text"
-      ]
-    },
-    "nullable": [
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true
-    ]
-  },
-  "hash": "f0fdeb7aea3e71099e7db0f4343bbd7ec86610ddc8589bf5b606fab0947c8b75"
-}

backend/.sqlx/query-f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7.json

@@ -0,0 +1,246 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n                            cj.id AS \"id!\",\n                            cj.workspace_id AS \"workspace_id!\",\n                            cj.parent_job,\n                            cj.created_by AS \"created_by!\",\n                            cj.duration_ms AS \"duration_ms!\",\n                            cj.success AS \"success!\",\n                            cj.script_hash AS \"script_hash!: Option<ScriptHash>\",\n                            cj.script_path,\n                            cj.args AS \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\",\n                            cj.result AS \"result: sqlx::types::Json<Box<RawValue>>\",\n                            cj.deleted AS \"deleted!\",\n                            cj.canceled AS \"canceled!\",\n                            cj.canceled_by,\n                            cj.canceled_reason,\n                            cj.job_kind AS \"job_kind!: JobKind\",\n                            cj.schedule_path,\n                            cj.permissioned_as AS \"permissioned_as!\",\n                            cj.is_flow_step AS \"is_flow_step!\",\n                            cj.language AS \"language: ScriptLang\",\n                            cj.is_skipped AS \"is_skipped!\",\n                            cj.email AS \"email!\",\n                            cj.visible_to_owner AS \"visible_to_owner!\",\n                            cj.mem_peak,\n                            cj.tag AS \"tag!\",\n                            cj.created_at AS \"created_at!\",\n                            cj.started_at,\n                            job_logs.logs,\n                            job_logs.log_offset AS \"log_offset?\",\n                            job_logs.log_file_index\n\n                        FROM v2_as_completed_job AS cj\n                        LEFT JOIN job_logs ON cj.id = job_logs.job_id\n                        WHERE cj.created_at < $1\n                        ORDER BY cj.created_at ASC LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id!",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 1,
+        "name": "workspace_id!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "parent_job",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 3,
+        "name": "created_by!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 4,
+        "name": "duration_ms!",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 5,
+        "name": "success!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 6,
+        "name": "script_hash!: Option<ScriptHash>",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 7,
+        "name": "script_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 8,
+        "name": "args: sqlx::types::Json<HashMap<String, Box<RawValue>>>",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 9,
+        "name": "result: sqlx::types::Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 10,
+        "name": "deleted!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 11,
+        "name": "canceled!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 12,
+        "name": "canceled_by",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 13,
+        "name": "canceled_reason",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 14,
+        "name": "job_kind!: JobKind",
+        "type_info": {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies",
+                "deploymentcallback",
+                "singlescriptflow",
+                "flowscript",
+                "flownode",
+                "appscript"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 15,
+        "name": "schedule_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 16,
+        "name": "permissioned_as!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 17,
+        "name": "is_flow_step!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 18,
+        "name": "language: ScriptLang",
+        "type_info": {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell",
+                "mssql",
+                "php",
+                "bunnative",
+                "rust",
+                "ansible",
+                "csharp",
+                "oracledb",
+                "nu",
+                "java"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 19,
+        "name": "is_skipped!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 20,
+        "name": "email!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 21,
+        "name": "visible_to_owner!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 22,
+        "name": "mem_peak",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 23,
+        "name": "tag!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 24,
+        "name": "created_at!",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 25,
+        "name": "started_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 26,
+        "name": "logs",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 27,
+        "name": "log_offset?",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 28,
+        "name": "log_file_index",
+        "type_info": "TextArray"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Timestamptz",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      false,
+      true
+    ]
+  },
+  "hash": "f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7"
+}

backend/.sqlx/query-f632d08a8d3df691fff9f57fdf926f787287c3cd181a6853056077edba10473d.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                SELECT EXISTS(\n                    SELECT 1 \n                    FROM \n                        http_trigger \n                    WHERE \n                        workspace_id = $1 AND \n                        path = $2\n                )\n                ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "exists",
-        "type_info": "Bool"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "f632d08a8d3df691fff9f57fdf926f787287c3cd181a6853056077edba10473d"
-}

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.490.0"
+version = "1.491.5"
 authors.workspace = true
 edition.workspace = true
 
@@ -32,7 +32,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.490.0"
+version = "1.491.5"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 

backend/ee-repo-ref.txt

@@ -1 +1 @@
-4dc1f25f4fcc013334d4cc1d07cbe60a22b56d1f
+3efa7fa51e9f93f60e141fef5b8b9338528cf955

backend/migrations/20250514120017_http_trigger_update_notify.down.sql

@@ -0,0 +1,4 @@
+-- Add down migration script here
+DROP TRIGGER http_trigger_change_trigger ON http_trigger;
+DROP FUNCTION notify_http_trigger_change();
+DROP SEQUENCE http_trigger_version_seq;

backend/migrations/20250514120017_http_trigger_update_notify.up.sql

@@ -0,0 +1,15 @@
+-- Add up migration script here
+CREATE OR REPLACE FUNCTION notify_http_trigger_change()
+RETURNS TRIGGER AS $$
+BEGIN
+    PERFORM pg_notify('notify_http_trigger_change', NEW.workspace_id || ':' || NEW.path);
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+CREATE TRIGGER http_trigger_change_trigger
+AFTER INSERT OR UPDATE OR DELETE ON http_trigger
+FOR EACH ROW
+EXECUTE FUNCTION notify_http_trigger_change();
+
+CREATE SEQUENCE http_trigger_version_seq;
+

backend/migrations/20250515155403_add_grant_on_dependency_map.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20250515155403_add_grant_on_dependency_map.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+GRANT ALL ON dependency_map TO windmill_user;
+GRANT ALL ON dependency_map TO windmill_admin;

backend/migrations/20250515181903_fix_http_routers_cache.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20250515181903_fix_http_routers_cache.up.sql

@@ -0,0 +1,9 @@
+-- Add up migration script here
+-- this makes sure that the first time nextval is called, 2 is returned
+-- otherwise, `SELECT last_value from http_trigger_version_seq;` would return 1 before and after the first nextval call
+-- which would not refresh the routers cache after the first create/update/delete
+SELECT setval(                                  
+    'http_trigger_version_seq',
+    (SELECT last_value FROM http_trigger_version_seq),
+    true
+);

backend/parsers/windmill-parser-sql/src/lib.rs

@@ -120,6 +120,60 @@ pub fn parse_db_resource(code: &str) -> Option<String> {
     cap.map(|x| x.get(1).map(|x| x.as_str().to_string()).unwrap())
 }
 
+#[derive(Clone, Copy, Debug)]
+pub enum S3ModeFormat {
+    Json,
+    Csv,
+    Parquet,
+}
+pub fn s3_mode_extension(format: S3ModeFormat) -> &'static str {
+    match format {
+        S3ModeFormat::Json => "json",
+        S3ModeFormat::Csv => "csv",
+        S3ModeFormat::Parquet => "parquet",
+    }
+}
+pub struct S3ModeArgs {
+    pub prefix: Option<String>,
+    pub storage: Option<String>,
+    pub format: S3ModeFormat,
+}
+pub fn parse_s3_mode(code: &str) -> anyhow::Result<Option<S3ModeArgs>> {
+    let cap = match RE_S3_MODE.captures(code) {
+        Some(x) => x,
+        None => return Ok(None),
+    };
+    let args_str = cap
+        .get(1)
+        .map(|x| x.as_str().to_string())
+        .unwrap_or_default();
+
+    let mut prefix = None;
+    let mut storage = None;
+    let mut format = S3ModeFormat::Json;
+
+    for kv in args_str.split(' ').map(|kv| kv.trim()) {
+        if kv.is_empty() {
+            continue;
+        }
+        let mut it = kv.split('=');
+        let (Some(key), Some(value)) = (it.next(), it.next()) else {
+            return Err(anyhow!("Invalid S3 mode argument: {}", kv));
+        };
+        match (key.trim(), value.trim()) {
+            ("prefix", _) => prefix = Some(value.to_string()),
+            ("storage", _) => storage = Some(value.to_string()),
+            ("format", "json") => format = S3ModeFormat::Json,
+            ("format", "parquet") => format = S3ModeFormat::Parquet,
+            ("format", "csv") => format = S3ModeFormat::Csv,
+            ("format", format) => return Err(anyhow!("Invalid S3 mode format: {}", format)),
+            (_, _) => return Err(anyhow!("Invalid S3 mode argument: {}", kv)),
+        }
+    }
+
+    Ok(Some(S3ModeArgs { prefix, storage, format }))
+}
+
 pub fn parse_sql_blocks(code: &str) -> Vec<&str> {
     let mut blocks = vec![];
     let mut last_idx = 0;
@@ -147,6 +201,7 @@ lazy_static::lazy_static! {
     static ref RE_NONEMPTY_SQL_BLOCK: Regex = Regex::new(r#"(?m)^\s*[^\s](?:[^-]|$)"#).unwrap();
 
     static ref RE_DB: Regex = Regex::new(r#"(?m)^-- database (\S+) *(?:\r|\n|$)"#).unwrap();
+    static ref RE_S3_MODE: Regex = Regex::new(r#"(?m)^-- s3( (.+))? *(?:\r|\n|$)"#).unwrap();
 
     // -- $1 name (type) = default
     static ref RE_ARG_MYSQL: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();

backend/src/main.rs

@@ -810,6 +810,21 @@ Windmill Community Edition {GIT_VERSION}
                                                         }
                                                     }
                                                 },
+                                                #[cfg(feature = "http_trigger")]
+                                                "notify_http_trigger_change" => {
+                                                    tracing::info!("HTTP trigger change detected: {}", n.payload());
+                                                    match windmill_api::http_triggers::refresh_routers(&db).await {
+                                                        Ok((true, _)) => {
+                                                            tracing::info!("Refreshed HTTP routers (trigger change)");
+                                                        },
+                                                        Ok((false, _)) => {
+                                                            tracing::warn!("Should have refreshed HTTP routers (trigger change) but did not");
+                                                        },
+                                                        Err(err) => {
+                                                            tracing::error!("Error refreshing HTTP routers (trigger change): {err:#}");
+                                                        }
+                                                    };
+                                                },
                                                 "notify_global_setting_change" => {
                                                     tracing::info!("Global setting change detected: {}", n.payload());
                                                     match n.payload() {
@@ -1133,6 +1148,10 @@ async fn listen_pg(url: &str) -> Option<PgListener> {
         "notify_workspace_envs_change",
         "notify_runnable_version_change",
     ];
+
+    #[cfg(feature = "http_trigger")]
+    channels.push("notify_http_trigger_change");
+
     #[cfg(feature = "cloud")]
     channels.push("notify_workspace_premium_change");
 

backend/src/monitor.rs

@@ -62,7 +62,11 @@ use windmill_common::{
     users::truncate_token,
     utils::{empty_as_none, now_from_db, rd_string, report_critical_error, Mode},
     worker::{
-        load_env_vars, load_init_bash_from_env, load_whitelist_env_vars_from_env, load_worker_config, reload_custom_tags_setting, store_pull_query, store_suspended_pull_query, update_min_version, Connection, WorkerConfig, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP
+        load_env_vars, load_init_bash_from_env, load_whitelist_env_vars_from_env,
+        load_worker_config, reload_custom_tags_setting, store_pull_query,
+        store_suspended_pull_query, update_min_version, Connection, WorkerConfig,
+        DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY,
+        SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP,
     },
     KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERT_MUTE_UI_ENABLED,
     CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS,
@@ -829,11 +833,7 @@ pub async fn delete_expired_items(db: &DB) -> () {
             Ok(mut tx) => {
                 let deleted_jobs = sqlx::query_scalar!(
                     "DELETE FROM v2_job_completed c
-                    USING v2_job j
-                    WHERE
-                        created_at <= now() - ($1::bigint::text || ' s')::interval
-                        AND completed_at + ($1::bigint::text || ' s')::interval <= now()
-                        AND c.id = j.id
+                    WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval 
                     RETURNING c.id",
                     job_retention_secs
                 )

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.490.0
+  version: 1.491.5
   title: Windmill API
 
   contact:
@@ -7899,6 +7899,52 @@ paths:
         "200":
           description: Interactive slack approval message sent successfully
 
+  /w/{workspace}/jobs/teams_approval/{id}:
+    get:
+      summary: generate interactive teams approval for suspended job
+      operationId: getTeamsApprovalPayload
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/JobId"
+        - name: approver
+          in: query
+          schema:
+            type: string
+        - name: message
+          in: query
+          schema:
+            type: string
+        - name: team_name
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: channel_name
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: flow_step_id
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: default_args_json
+          in: query
+          required: false
+          schema:
+            type: string
+        - name: dynamic_enums_json
+          in: query
+          required: false
+          schema:
+            type: string
+      responses:
+        "200":
+          description: Interactive slack approval message sent successfully
+
   /w/{workspace}/jobs_u/resume/{id}/{resume_id}/{signature}:
     get:
       summary: resume a job for a suspended flow
@@ -10982,7 +11028,8 @@ paths:
           description: a config
           content:
             application/json:
-              schema: {}
+              schema:
+                $ref: "#/components/schemas/Configs"
 
   /configs/update/{name}:
     post:
@@ -13137,6 +13184,37 @@ components:
         code_completion_model:
           $ref: "#/components/schemas/AIProviderModel"
 
+    Alert:
+      type: object
+      properties:
+        name:
+          type: string
+        tags_to_monitor:
+          type: array
+          items:
+            type: string
+        jobs_num_threshold:
+          type: integer
+        alert_cooldown_seconds:
+          type: integer
+        alert_time_threshold_seconds:
+          type: integer
+      required:
+        - name
+        - tags_to_monitor
+        - jobs_num_threshold
+        - alert_cooldown_seconds
+        - alert_time_threshold_seconds
+
+    Configs:
+      type: object
+      nullable: true
+      properties:
+        alerts:
+          type: array
+          items:
+            $ref: '#/components/schemas/Alert'
+
     Script:
       type: object
       properties:

backend/windmill-api/src/agent_workers_ee.rs

@@ -28,7 +28,7 @@ pub fn workspaced_service(
     use windmill_worker::JobCompletedSender;
 
     let (job_completed_tx, _job_completed_rx) =
-        JobCompletedSender::new(&Connection::Sql(db.clone()), 100);
+        JobCompletedSender::new(&Connection::Sql(db.clone()), 10);
 
     let router = Router::new();
 

backend/windmill-api/src/approvals.rs

@@ -0,0 +1,292 @@
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use uuid::Uuid;
+use std::str::FromStr;
+use regex::Regex;
+use serde_json::Value;
+use crate::db::{ApiAuthed, DB};
+use crate::jobs::{cancel_suspended_job, resume_suspended_job, QueryApprover, QueryOrBody, ResumeUrls, get_resume_urls_internal};
+use axum::{extract::{Path, Query}, Extension};
+use windmill_common::error::Error;
+use windmill_common::cache;
+use windmill_common::jobs::JobKind;
+use windmill_common::scripts::ScriptHash;
+use serde_json::value::RawValue;
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct ResumeSchema {
+    pub schema: Schema,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct Schema {
+    pub order: Vec<String>,
+    pub required: Vec<String>,
+    pub properties: HashMap<String, ResumeFormField>,
+}
+
+#[derive(Debug, Deserialize, Serialize, Clone)]
+#[serde(rename_all = "lowercase")]
+pub enum FieldType {
+    Boolean,
+    String,
+    Number,
+    Integer,
+    #[serde(other)]
+    Unknown,
+}
+
+#[derive(Debug, Deserialize, Serialize, Clone)]
+pub struct ResumeFormField {
+    pub r#type: FieldType,
+    pub format: Option<String>,
+    pub default: Option<serde_json::Value>,
+    pub description: Option<String>,
+    pub title: Option<String>,
+    pub r#enum: Option<Vec<String>>,
+    #[serde(rename = "enumLabels")]
+    pub enum_labels: Option<HashMap<String, String>>,
+    pub nullable: Option<bool>,
+    pub placeholder: Option<String>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct ResumeFormRow {
+    pub resume_form: Option<serde_json::Value>,
+    pub hide_cancel: Option<bool>,
+}
+
+#[derive(Deserialize)]
+pub struct QueryMessage {
+    pub message: Option<String>,
+}
+
+#[derive(Deserialize)]
+pub struct QueryFlowStepId {
+    pub flow_step_id: String,
+}
+
+#[derive(Deserialize, Debug)]
+pub struct QueryDefaultArgsJson {
+    pub default_args_json: Option<serde_json::Value>,
+}
+
+#[derive(Deserialize, Debug)]
+pub struct QueryDynamicEnumJson {
+    pub dynamic_enums_json: Option<serde_json::Value>,
+}
+
+#[derive(Debug)]
+pub struct ApprovalFormDetails {
+    pub message_str: String,
+    pub urls: ResumeUrls,
+    pub schema: Option<ResumeFormRow>,
+}
+
+#[allow(dead_code)]
+#[derive(Debug, Clone, Copy)]
+pub enum MessageFormat {
+    Slack,
+    Teams,
+}
+
+pub fn extract_w_id_from_resume_url(resume_url: &str) -> Result<&str, Error> {
+    let re = Regex::new(r"/api/w/(?P<w_id>[^/]+)/jobs_u/(?P<action>resume|cancel)/(?P<job_id>[^/]+)/(?P<resume_id>[^/]+)/(?P<secret>[a-fA-F0-9]+)(?:\?approver=(?P<approver>[^&]+))?").unwrap();
+    let captures = re.captures(resume_url).ok_or_else(|| {
+        tracing::error!("Resume URL does not match the pattern.");
+        Error::BadRequest("Invalid URL format.".to_string())
+    })?;
+    Ok(captures.name("w_id").map_or("", |m| m.as_str()))
+}
+
+pub async fn handle_resume_action(
+    authed: Option<ApiAuthed>,
+    db: DB,
+    resume_url: &str,
+    form_data: Value,
+    action: &str,
+) -> Result<(), Error> {
+    // Extract information from resume_url using regex
+    let re = Regex::new(r"/api/w/(?P<w_id>[^/]+)/jobs_u/(?P<action>resume|cancel)/(?P<job_id>[^/]+)/(?P<resume_id>[^/]+)/(?P<secret>[a-fA-F0-9]+)(?:\?approver=(?P<approver>[^&]+))?").unwrap();
+    let captures = re.captures(resume_url).ok_or_else(|| {
+        tracing::error!("Resume URL does not match the pattern.");
+        Error::BadRequest("Invalid URL format.".to_string())
+    })?;
+
+    let (w_id, job_id, resume_id, secret, approver) = (
+        captures.name("w_id").map_or("", |m| m.as_str()),
+        captures.name("job_id").map_or("", |m| m.as_str()),
+        captures.name("resume_id").map_or("", |m| m.as_str()),
+        captures.name("secret").map_or("", |m| m.as_str()),
+        captures.name("approver").map(|m| m.as_str().to_string()),
+    );
+
+    let approver = QueryApprover { approver };
+
+    // Convert job_id and resume_id to appropriate types
+    let job_uuid = Uuid::from_str(job_id)
+        .map_err(|_| Error::BadRequest("Invalid job ID format.".to_string()))?;
+
+    let resume_id_parsed = resume_id
+        .parse::<u32>()
+        .map_err(|_| Error::BadRequest("Invalid resume ID format.".to_string()))?;
+
+    // Call the appropriate function based on the action
+    let res = if action == "resume" {
+        resume_suspended_job(
+            authed,
+            Extension(db.clone()),
+            Path((
+                w_id.to_string(),
+                job_uuid,
+                resume_id_parsed,
+                secret.to_string(),
+            )),
+            Query(approver),
+            QueryOrBody(Some(form_data)),
+        )
+        .await
+    } else {
+        cancel_suspended_job(
+            authed,
+            Extension(db.clone()),
+            Path((
+                w_id.to_string(),
+                job_uuid,
+                resume_id_parsed,
+                secret.to_string(),
+            )),
+            Query(approver),
+            QueryOrBody(Some(form_data)),
+        )
+        .await
+    };
+
+    tracing::debug!("Job action result: {:#?}", res);
+    res?;
+
+    Ok(())
+}
+
+pub async fn get_approval_form_details(
+    db: DB,
+    w_id: &str,
+    job_id: Uuid,
+    flow_step_id: Option<&str>,
+    resume_id: u32,
+    approver: Option<&str>,
+    message: Option<&str>,
+    format: MessageFormat,
+) -> Result<ApprovalFormDetails, Error> {
+    let res = get_resume_urls_internal(
+        axum::Extension(db.clone()),
+        Path((w_id.to_string(), job_id, resume_id)),
+        Query(QueryApprover { approver: approver.map(|a| a.to_string()) }),
+    )
+    .await?;
+
+    let urls = res.0;
+
+    tracing::debug!("Job ID: {:?}", job_id);
+
+    // TODO: do we have a helper function for this?
+    let (job_kind, script_hash, raw_flow, parent_job_id, created_at, created_by, script_path, args) = sqlx::query!(
+        "WITH job_info AS (
+            -- Query for Teams (running jobs)
+            SELECT
+                parent.job_kind AS \"job_kind!: JobKind\",
+                parent.script_hash AS \"script_hash: ScriptHash\",
+                parent.raw_flow AS \"raw_flow: sqlx::types::Json<Box<RawValue>>\",
+                child.parent_job AS \"parent_job: Uuid\",
+                parent.created_at AS \"created_at!: chrono::NaiveDateTime\",
+                parent.created_by AS \"created_by!\",
+                parent.script_path,
+                parent.args AS \"args: sqlx::types::Json<Box<RawValue>>\"
+            FROM v2_as_queue child
+            JOIN v2_as_queue parent ON parent.id = child.parent_job
+            WHERE child.id = $1 AND child.workspace_id = $2
+            UNION ALL
+            -- Query for Slack (completed jobs)
+            SELECT
+                v2_as_queue.job_kind AS \"job_kind!: JobKind\",
+                v2_as_queue.script_hash AS \"script_hash: ScriptHash\",
+                v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json<Box<RawValue>>\",
+                v2_as_completed_job.parent_job AS \"parent_job: Uuid\",
+                v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\",
+                v2_as_completed_job.created_by AS \"created_by!\",
+                v2_as_queue.script_path,
+                v2_as_queue.args AS \"args: sqlx::types::Json<Box<RawValue>>\"
+            FROM v2_as_queue
+            JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id
+            WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2
+        )
+        SELECT * FROM job_info LIMIT 1",
+        job_id,
+        &w_id
+    )
+    .fetch_optional(&db)
+    .await
+    .map_err(|e| Error::BadRequest(e.to_string()))?
+    .ok_or_else(|| Error::BadRequest("This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string()))
+    .map(|r| (r.job_kind, r.script_hash, r.raw_flow, r.parent_job, r.created_at, r.created_by, r.script_path, r.args))?;
+
+    let flow_data = match cache::job::fetch_flow(&db, job_kind, script_hash).await {
+        Ok(data) => data,
+        Err(_) => {
+            if let Some(parent_job_id) = parent_job_id.as_ref() {
+                cache::job::fetch_preview_flow(&db, parent_job_id, raw_flow).await?
+            } else {
+                return Err(Error::BadRequest(
+                    "This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string(),
+                ));
+            }
+        }
+    };
+
+    let flow_value = &flow_data.flow;
+    let flow_step_id = flow_step_id.unwrap_or("");
+    let module = flow_value.modules.iter().find(|m| m.id == flow_step_id);
+
+    tracing::debug!("Module: {:#?}", module);
+
+    let schema = module.and_then(|module| {
+        module.suspend.as_ref().map(|suspend| ResumeFormRow {
+            resume_form: suspend.resume_form.clone(),
+            hide_cancel: suspend.hide_cancel,
+        })
+    });
+
+    let args_str = args.map_or("None".to_string(), |a| a.get().to_string());
+    let parent_job_id_str = parent_job_id.map_or("None".to_string(), |id| id.to_string());
+    let script_path_str = script_path.as_deref().unwrap_or("None");
+
+    let created_at_formatted = created_at.format("%Y-%m-%d %H:%M:%S").to_string();
+
+    let bold_format = match format {
+        MessageFormat::Slack => "*{}*",
+        MessageFormat::Teams => "**{}**",
+    };
+
+    let mut message_str = format!(
+        "A workflow has been suspended and is waiting for approval:\n\n\
+        {}: {created_by}\n\n\
+        {}: {created_at_formatted}\n\n\
+        {}: {script_path_str}\n\n\
+        {}: {args_str}\n\n\
+        {}: {parent_job_id_str}\n\n",
+        bold_format.replace("{}", "Created by"),
+        bold_format.replace("{}", "Created at"),
+        bold_format.replace("{}", "Script path"),
+        bold_format.replace("{}", "Args"),
+        bold_format.replace("{}", "Flow ID")
+    );
+
+    // Append custom message if provided
+    if let Some(msg) = message {
+        message_str.push_str(msg);
+    }
+
+    tracing::debug!("Schema: {:#?}", schema);
+
+    Ok(ApprovalFormDetails { message_str, urls, schema })
+}

backend/windmill-api/src/args.rs

@@ -29,14 +29,14 @@ pub enum RawBody {
     Empty,
 }
 
-#[derive(Clone, Serialize)]
+#[derive(Debug, Clone, Serialize)]
 #[serde(untagged)]
 pub enum Body {
     HashMap(HashMap<String, Box<RawValue>>),
     NoHashMap(Box<RawValue>),
 }
 
-#[derive(Clone, Default)]
+#[derive(Debug, Clone, Default)]
 pub struct WebhookArgsMetadata {
     pub raw_string: Option<String>,
     pub headers: HashMap<String, Box<RawValue>>,
@@ -51,7 +51,7 @@ pub struct RawWebhookArgs {
     pub metadata: WebhookArgsMetadata,
 }
 
-#[derive(Clone)]
+#[derive(Debug, Clone)]
 pub struct WebhookArgs {
     pub body: Body,
     pub metadata: WebhookArgsMetadata,

backend/windmill-api/src/capture.rs

@@ -14,11 +14,18 @@ use {
 };
 
 #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
-use crate::gcp_triggers_ee::{
-    manage_google_subscription, process_google_push_request, validate_jwt_token,
-    CreateUpdateConfig, SubscriptionMode,
+use {
+    crate::gcp_triggers_ee::{
+        manage_google_subscription, process_google_push_request, validate_jwt_token,
+        CreateUpdateConfig, SubscriptionMode,
+    },
+    axum::extract::Request,
+    http::HeaderMap,
 };
 
+#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
+use windmill_common::utils::empty_as_none;
+
 #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
 use windmill_common::auth::aws::AwsAuthResourceType;
 
@@ -26,12 +33,7 @@ use windmill_common::auth::aws::AwsAuthResourceType;
     feature = "http_trigger",
     all(feature = "enterprise", feature = "gcp_trigger")
 ))]
-use {
-    axum::extract::Request,
-    http::HeaderMap,
-    serde::de::DeserializeOwned,
-    windmill_common::{error::Error, utils::empty_as_none},
-};
+use {serde::de::DeserializeOwned, windmill_common::error::Error};
 
 #[cfg(all(feature = "enterprise", feature = "kafka"))]
 use crate::kafka_triggers_ee::KafkaTriggerConfigConnection;
@@ -381,6 +383,7 @@ async fn set_gcp_trigger_config(
         gcp_config.subscription_mode,
         gcp_config.create_update,
         false,
+        capture_config.is_flow,
     )
     .await?;
     gcp_config.create_update = Some(config);

backend/windmill-api/src/db.rs

@@ -782,10 +782,32 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> {
         .execute(db)
         .await?;
     });
+
+    run_windmill_migration!("job_completed_completed_at", db, |tx| {
+        sqlx::query!(
+            "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_job_completed_completed_at ON v2_job_completed (completed_at DESC)"
+        )
+        .execute(db)
+        .await?;
+    });
+
+    run_windmill_migration!("alerts_by_workspace", db, |tx| {
+        sqlx::query!(
+            "CREATE INDEX CONCURRENTLY IF NOT EXISTS alerts_by_workspace ON alerts (workspace_id);"
+        )
+        .execute(db)
+        .await?;
+    });
+
+    run_windmill_migration!("remove_redundant_log_file_index", db, |tx| {
+        sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS log_file_hostname_log_ts_idx")
+            .execute(db)
+            .await?;
+    });
     Ok(())
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Hash, Eq, PartialEq)]
 pub struct ApiAuthed {
     pub email: String,
     pub username: String,

backend/windmill-api/src/gcp_triggers_ee.rs

@@ -86,6 +86,7 @@ pub async fn manage_google_subscription(
     _subscription_mode: SubscriptionMode,
     _create_update_config: Option<CreateUpdateConfig>,
     _trigger_mode: bool,
+    _is_flow: bool
 ) -> WindmillResult<CreateUpdateConfig> {
     Ok(CreateUpdateConfig::default())
 }

backend/windmill-api/src/http_trigger_args.rs

@@ -17,7 +17,7 @@ use crate::{
 
 pub struct RawHttpTriggerArgs(pub RawWebhookArgs);
 
-#[derive(Serialize, Deserialize, sqlx::Type, Debug)]
+#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Hash, Eq, PartialEq)]
 #[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")]
 #[serde(rename_all = "lowercase")]
 pub enum HttpMethod {
@@ -56,7 +56,7 @@ where
     }
 }
 
-#[derive(Clone)]
+#[derive(Debug, Clone)]
 pub struct HttpTriggerArgs(pub WebhookArgs);
 
 impl RawHttpTriggerArgs {

backend/windmill-api/src/http_trigger_auth.rs

@@ -414,7 +414,7 @@ pub enum Encoding {
     Base64Uri,
     Hex,
 }
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct SignatureAuthenticationMethod {
     algorithm: HmacAlgorithm,
     encoding: Encoding,
@@ -426,20 +426,20 @@ pub struct SignatureConfigData<'config> {
     secret_key: &'config str,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct SignatureAuthentication {
     signature_provider: WebhookType,
     secret_key: String,
     authentication_config: Option<SignatureAuthenticationMethod>,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct BasicAuthAuthentication {
     pub username: String,
     pub password: String,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct ApiKeyAuthentication {
     api_key_header: String,
     api_key_secret: String,
@@ -558,7 +558,7 @@ pub fn verify_hmac_signature(
     Ok(())
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 #[serde(untagged)]
 pub enum AuthenticationMethod {
     Signature(SignatureAuthentication),

backend/windmill-api/src/http_triggers.rs

@@ -4,7 +4,7 @@ use crate::http_trigger_args::{HttpMethod, RawHttpTriggerArgs};
 use crate::job_helpers_ee::get_workspace_s3_resource;
 use crate::resources::try_get_resource_from_db_as;
 use crate::trigger_helpers::{get_runnable_format, RunnableId};
-use crate::utils::non_empty_str;
+use crate::utils::{non_empty_str, ExpiringCacheEntry};
 use crate::{
     auth::{AuthCache, OptTokened},
     db::{ApiAuthed, DB},
@@ -24,11 +24,14 @@ use axum::{
 #[cfg(feature = "parquet")]
 use http::header::IF_NONE_MATCH;
 use http::{HeaderMap, StatusCode};
+use quick_cache::sync::Cache;
 use serde::{Deserialize, Serialize};
 use sql_builder::{bind::Bind, SqlBuilder};
 use sqlx::prelude::FromRow;
+use sqlx::PgTransaction;
 use std::borrow::Cow;
 use std::{collections::HashMap, sync::Arc};
+use tokio::sync::{RwLock, RwLockReadGuard};
 use tower_http::cors::CorsLayer;
 use windmill_audit::{audit_ee::audit_log, ActionKind};
 use windmill_common::error::Error;
@@ -283,6 +286,16 @@ fn validate_authentication_method(
     Ok(())
 }
 
+async fn increase_trigger_version_and_commit(mut tx: PgTransaction<'_>) -> error::Result<()> {
+    sqlx::query!("SELECT nextval('http_trigger_version_seq')",)
+        .fetch_one(&mut *tx)
+        .await?;
+
+    tx.commit().await?;
+
+    Ok(())
+}
+
 async fn create_trigger(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
@@ -382,7 +395,7 @@ async fn create_trigger(
     )
     .await?;
 
-    tx.commit().await?;
+    increase_trigger_version_and_commit(tx).await?;
 
     Ok((StatusCode::CREATED, format!("{}", ct.path)))
 }
@@ -538,7 +551,7 @@ async fn update_trigger(
     )
     .await?;
 
-    tx.commit().await?;
+    increase_trigger_version_and_commit(tx).await?;
 
     Ok(path.to_string())
 }
@@ -572,7 +585,7 @@ async fn delete_trigger(
     )
     .await?;
 
-    tx.commit().await?;
+    increase_trigger_version_and_commit(tx).await?;
 
     Ok(format!("HTTP trigger {path} deleted"))
 }
@@ -685,8 +698,8 @@ async fn exists_route(
     Ok(Json(exists))
 }
 
-#[derive(Debug, Deserialize)]
-struct TriggerRoute {
+#[derive(Debug, Deserialize, Clone)]
+pub struct TriggerRoute {
     path: String,
     script_path: String,
     is_flow: bool,
@@ -704,6 +717,145 @@ struct TriggerRoute {
     raw_string: bool,
 }
 
+pub struct RoutersCache {
+    routers: HashMap<HttpMethod, matchit::Router<TriggerRoute>>,
+    version: i64,
+}
+
+lazy_static::lazy_static! {
+    static ref HTTP_ACCESS_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry<()>> = Cache::new(100);
+    static ref HTTP_AUTH_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry<crate::http_trigger_auth::AuthenticationMethod>> = Cache::new(100);
+
+    static ref HTTP_ROUTERS_CACHE: RwLock<RoutersCache> = RwLock::new(RoutersCache {
+        routers: HashMap::new(),
+        version: 0,
+    });
+}
+
+pub async fn refresh_routers_loop(
+    db: &DB,
+    mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
+) -> () {
+    match refresh_routers(db).await {
+        Ok(_) => {
+            tracing::info!("Loaded HTTP routers");
+        }
+        Err(err) => {
+            tracing::error!("Error loading HTTP routers: {err:#}");
+        }
+    };
+    let db = db.clone();
+    tokio::spawn(async move {
+        loop {
+            tokio::select! {
+                _ = killpill_rx.recv() => {
+                    break;
+                }
+                _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => {
+                    match refresh_routers(&db).await {
+                        Ok((true, _)) => {
+                            tracing::info!("Refreshed HTTP routers");
+                        }
+                        Err(err) => {
+                            tracing::error!("Error refreshing HTTP routers: {err:#}");
+                        }
+                        _ => {}
+                    }
+                }
+            }
+        }
+    });
+}
+
+pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, RoutersCache>), Error> {
+    let version = sqlx::query_scalar!("SELECT last_value FROM http_trigger_version_seq",)
+        .fetch_one(db)
+        .await?;
+    let routers_cache = HTTP_ROUTERS_CACHE.read().await;
+    if routers_cache.version == 0 || version > routers_cache.version {
+        drop(routers_cache);
+        let mut routers = HashMap::new();
+
+        for http_method in [
+            HttpMethod::Get,
+            HttpMethod::Post,
+            HttpMethod::Put,
+            HttpMethod::Patch,
+            HttpMethod::Delete,
+        ] {
+            let triggers = sqlx::query_as!(
+                TriggerRoute,
+                r#"
+                    SELECT 
+                        path, 
+                        script_path, 
+                        is_flow, 
+                        route_path, 
+                        authentication_resource_path,
+                        workspace_id, 
+                        is_async, 
+                        authentication_method  AS "authentication_method: _", 
+                        edited_by, 
+                        email, 
+                        static_asset_config AS "static_asset_config: _",
+                        wrap_body,
+                        raw_string,
+                        workspaced_route,
+                        is_static_website
+                    FROM 
+                        http_trigger 
+                    WHERE 
+                        http_method = $1
+                    "#,
+                &http_method as &HttpMethod
+            )
+            .fetch_all(db)
+            .await?;
+
+            let mut router = matchit::Router::new();
+
+            for trigger in triggers {
+                let full_path = if trigger.workspaced_route || *CLOUD_HOSTED {
+                    format!("/{}/{}", trigger.workspace_id, trigger.route_path)
+                } else {
+                    format!("/{}", trigger.route_path)
+                };
+
+                if trigger.is_static_website {
+                    router
+                        .insert(format!("{}/*wm_subpath", full_path), trigger.clone())
+                        .unwrap_or_else(|e| {
+                            tracing::warn!(
+                                "Failed to consider http trigger route {}/*wm_subpath: {:?}",
+                                full_path,
+                                e,
+                            );
+                        });
+                }
+                router
+                    .insert(full_path.clone(), trigger.clone())
+                    .unwrap_or_else(|e| {
+                        tracing::warn!(
+                            "Failed to consider http trigger route {}: {:?}",
+                            full_path,
+                            e,
+                        );
+                    });
+            }
+
+            routers.insert(http_method, router);
+        }
+
+        let mut routers_cache = HTTP_ROUTERS_CACHE.write().await;
+        *routers_cache = RoutersCache { routers, version };
+
+        Ok((true, routers_cache.downgrade()))
+    } else {
+        tracing::debug!("No HTTP routers refresh needed");
+        Ok((false, routers_cache))
+    }
+}
+
 async fn get_http_route_trigger(
     route_path: &str,
     auth_cache: &Arc<AuthCache>,
@@ -713,111 +865,30 @@ async fn get_http_route_trigger(
     method: &http::Method,
 ) -> error::Result<(TriggerRoute, String, HashMap<String, String>, ApiAuthed)> {
     let http_method: HttpMethod = method.try_into()?;
-    let (mut triggers, route_path) = if *CLOUD_HOSTED {
-        let mut splitted = route_path.split("/");
-        let w_id = splitted.next().ok_or_else(|| {
-            error::Error::BadRequest("Missing workspace id in route path".to_string())
-        })?;
-        let route_path = StripPath(splitted.collect::<Vec<_>>().join("/"));
-        let triggers = sqlx::query_as!(
-            TriggerRoute,
-            r#"
-            SELECT 
-                path, 
-                script_path, 
-                is_flow, 
-                route_path, 
-                workspace_id, 
-                is_async, 
-                authentication_method  AS "authentication_method: _", 
-                edited_by, 
-                email,
-                static_asset_config AS "static_asset_config: _",
-                wrap_body,
-                raw_string,
-                workspaced_route,
-                is_static_website,
-                authentication_resource_path
-            FROM 
-                http_trigger 
-            WHERE 
-                workspace_id = $1 AND 
-                http_method = $2
-            "#,
-            w_id,
-            http_method as HttpMethod
-        )
-        .fetch_all(db)
-        .await?;
-        (triggers, route_path)
+
+    let requested_path = format!("/{}", route_path);
+
+    let routers_cache = HTTP_ROUTERS_CACHE.read().await;
+
+    let routers_cache = if routers_cache.routers.is_empty() {
+        tracing::warn!("HTTP routers are not loaded, loading from db");
+        let (_, routers_cache) = refresh_routers(db).await?;
+        routers_cache
     } else {
-        let triggers = sqlx::query_as!(
-            TriggerRoute,
-            r#"
-            SELECT 
-                path, 
-                script_path, 
-                is_flow, 
-                route_path, 
-                authentication_resource_path,
-                workspace_id, 
-                is_async, 
-                authentication_method  AS "authentication_method: _", 
-                edited_by, 
-                email, 
-                static_asset_config AS "static_asset_config: _",
-                wrap_body,
-                raw_string,
-                workspaced_route,
-                is_static_website
-            FROM 
-                http_trigger 
-            WHERE 
-                http_method = $1
-            "#,
-            http_method as HttpMethod
-        )
-        .fetch_all(db)
-        .await?;
-        (triggers, StripPath(route_path.to_string()))
+        routers_cache
     };
 
-    let mut router = matchit::Router::new();
+    let router = routers_cache
+        .routers
+        .get(&http_method)
+        .ok_or(error::Error::internal_err(
+            "HTTP routers could not be loaded".to_string(),
+        ))?;
 
-    for (idx, trigger) in triggers.iter().enumerate() {
-        let route_path = match trigger.workspaced_route {
-            true => format!("{}/{}", &trigger.workspace_id, &trigger.route_path),
-            _ => trigger.route_path.clone(),
-        };
-        if trigger.is_static_website {
-            router
-                .insert(format!("/{}/*wm_subpath", route_path), idx)
-                .unwrap_or_else(|e| {
-                    tracing::warn!(
-                        "Failed to consider http trigger route {}: {:?}",
-                        route_path,
-                        e,
-                    );
-                });
-        }
-        router
-            .insert(format!("/{}", route_path), idx)
-            .unwrap_or_else(|e| {
-                tracing::warn!(
-                    "Failed to consider http trigger route {}: {:?}",
-                    route_path,
-                    e,
-                );
-            });
-    }
+    let trigger_match = router.at(requested_path.as_str()).ok();
 
-    let requested_path = format!("/{}", route_path.0);
-    let trigger_idx = router.at(requested_path.as_str()).ok();
-
-    let matchit::Match { value: trigger_idx, params } =
-        not_found_if_none(trigger_idx, "Trigger", requested_path.as_str())?;
-
-    let trigger = triggers.remove(trigger_idx.to_owned());
+    let matchit::Match { value: trigger, params } =
+        not_found_if_none(trigger_match, "Trigger", requested_path.as_str())?;
 
     let params: HashMap<String, String> = params
         .iter()
@@ -834,25 +905,49 @@ async fn get_http_route_trigger(
         };
         if let Some(authed) = opt_authed {
             // check that the user has access to the trigger
-            let mut tx = user_db.begin(&authed).await?;
-            let exists = sqlx::query_scalar!(
-                r#"
-                SELECT EXISTS(
-                    SELECT 1 
-                    FROM 
-                        http_trigger 
-                    WHERE 
-                        workspace_id = $1 AND 
-                        path = $2
-                )
-                "#,
-                trigger.workspace_id,
-                trigger.path
-            )
-            .fetch_one(&mut *tx)
-            .await?
-            .unwrap_or(false);
-            tx.commit().await?;
+            let cache_key = (
+                trigger.workspace_id.clone(),
+                trigger.path.clone(),
+                authed.clone(),
+            );
+            let exists = match HTTP_ACCESS_CACHE.get(&cache_key) {
+                Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => {
+                    tracing::debug!("HTTP access cache hit for trigger {}", trigger.path);
+                    true
+                }
+                _ => {
+                    tracing::debug!("HTTP access cache miss for trigger {}", trigger.path);
+                    let mut tx = user_db.begin(&authed).await?;
+                    let exists = sqlx::query_scalar!(
+                        r#"
+                        SELECT EXISTS(
+                            SELECT 1
+                            FROM 
+                                http_trigger 
+                            WHERE 
+                                workspace_id = $1 AND 
+                                path = $2
+                        )
+                        "#,
+                        trigger.workspace_id,
+                        trigger.path
+                    )
+                    .fetch_one(&mut *tx)
+                    .await?
+                    .unwrap_or(false);
+                    if exists {
+                        HTTP_ACCESS_CACHE.insert(
+                            cache_key,
+                            ExpiringCacheEntry {
+                                value: (),
+                                expiry: std::time::Instant::now()
+                                    + std::time::Duration::from_secs(10),
+                            },
+                        );
+                    }
+                    exists
+                }
+            };
             if exists {
                 Some(authed.display_username().to_owned())
             } else {
@@ -876,7 +971,7 @@ async fn get_http_route_trigger(
     )
     .await?;
 
-    Ok((trigger, route_path.0, params, authed))
+    Ok((trigger.clone(), route_path.to_string(), params, authed))
 }
 
 async fn route_job(
@@ -901,7 +996,15 @@ async fn route_job(
     .map_err(|e| e.into_response())?;
 
     let args = args
-        .process_args(&authed, &db, &trigger.workspace_id, trigger.raw_string)
+        .process_args(
+            &authed,
+            &db,
+            &trigger.workspace_id,
+            match trigger.authentication_method {
+                AuthenticationMethod::CustomScript | AuthenticationMethod::Signature => true,
+                _ => trigger.raw_string,
+            },
+        )
         .await
         .map_err(|e| e.into_response())?;
 
@@ -920,31 +1023,45 @@ async fn route_job(
                 }
             };
 
-            let authentication_method =
-                try_get_resource_from_db_as::<crate::http_trigger_auth::AuthenticationMethod>(
-                    authed.clone(),
-                    Some(user_db.clone()),
-                    &db,
-                    &resource_path,
-                    &trigger.workspace_id,
-                )
-                .await
-                .map_err(|e| e.into_response())?;
+            let cache_key = (
+                trigger.workspace_id.clone(),
+                resource_path.clone(),
+                authed.clone(),
+            );
 
-            let raw_payload = args
-                .0
-                .metadata
-                .raw_string
-                .as_ref()
-                .map(|raw_payload| serde_json::from_str::<String>(raw_payload))
-                .transpose()
-                .map_err(|e| {
-                    windmill_common::error::Error::SerdeJson { location: e.to_string(), error: e }
-                        .into_response()
-                })?;
+            let authentication_method = match HTTP_AUTH_CACHE.get(&cache_key) {
+                Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => {
+                    tracing::debug!("HTTP auth method cache hit for trigger {}", trigger.path);
+                    cache_entry.value
+                }
+                _ => {
+                    tracing::debug!("HTTP auth method cache miss for trigger {}", trigger.path);
+                    let auth_method = try_get_resource_from_db_as::<
+                        crate::http_trigger_auth::AuthenticationMethod,
+                    >(
+                        authed.clone(),
+                        Some(user_db.clone()),
+                        &db,
+                        &resource_path,
+                        &trigger.workspace_id,
+                    )
+                    .await
+                    .map_err(|e| e.into_response())?;
+                    HTTP_AUTH_CACHE.insert(
+                        cache_key,
+                        ExpiringCacheEntry {
+                            value: auth_method.clone(),
+                            expiry: std::time::Instant::now() + std::time::Duration::from_secs(60),
+                        },
+                    );
+                    auth_method
+                }
+            };
+
+            let raw_payload = args.0.metadata.raw_string.as_ref();
 
             let response = authentication_method
-                .authenticate_http_request(&headers, raw_payload.as_ref())
+                .authenticate_http_request(&headers, raw_payload)
                 .map_err(|e| e.into_response())?;
 
             if let Some(response) = response {

backend/windmill-api/src/jobs.rs

@@ -42,7 +42,7 @@ use crate::trigger_helpers::RunnableId;
 use crate::users::get_scope_tags;
 use crate::utils::content_plain;
 use crate::{
-    args::{DecodeQueries, RawWebhookArgs},
+    args::{self, RawWebhookArgs},
     db::DB,
     users::{check_scopes, require_owner_of_path, OptAuthed},
     utils::require_super_admin,
@@ -4330,7 +4330,7 @@ pub async fn run_wait_result_job_by_path_get(
     Extension(db): Extension<DB>,
     Path((w_id, script_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
-    DecodeQueries(queries): DecodeQueries,
+    args: RawWebhookArgs,
 ) -> error::Result<Response> {
     #[cfg(feature = "enterprise")]
     check_license_key_valid().await?;
@@ -4342,17 +4342,23 @@ pub async fn run_wait_result_job_by_path_get(
         x.map_err(|e| Error::internal_err(format!("Impossible to decode query payload: {e:#?}")))
     });
 
-    let mut payload_args = if let Some(payload) = payload_r {
+    let payload_args = if let Some(payload) = payload_r {
         payload?
     } else {
         HashMap::new()
     };
-    queries.iter().for_each(|(k, v)| {
-        payload_args.insert(k.to_string(), v.clone());
-    });
 
-    let inner_args: HashMap<String, Box<RawValue>> = HashMap::new();
-    let args = PushArgs { extra: Some(payload_args), args: &inner_args };
+    let mut args = args.process_args(&authed, &db, &w_id, None).await?;
+    args.body = args::Body::HashMap(payload_args);
+
+    let args = args
+        .to_args_from_runnable(
+            &db,
+            &w_id,
+            RunnableId::from_script_path(&script_path.0),
+            run_query.skip_preprocessor,
+        )
+        .await?;
 
     check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
     let script_path = script_path.to_path();
@@ -4424,7 +4430,7 @@ pub async fn run_wait_result_flow_by_path_get(
     Extension(db): Extension<DB>,
     Path((w_id, flow_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
-    DecodeQueries(queries): DecodeQueries,
+    args: RawWebhookArgs,
 ) -> error::Result<Response> {
     #[cfg(feature = "enterprise")]
     check_license_key_valid().await?;
@@ -4438,17 +4444,23 @@ pub async fn run_wait_result_flow_by_path_get(
         })
     });
 
-    let mut payload_args = if let Some(payload) = payload_r {
+    let payload_args = if let Some(payload) = payload_r {
         payload?
     } else {
         HashMap::new()
     };
 
-    queries.iter().for_each(|(k, v)| {
-        payload_args.insert(k.to_string(), v.clone());
-    });
+    let mut args = args.process_args(&authed, &db, &w_id, None).await?;
+    args.body = args::Body::HashMap(payload_args);
 
-    let args = PushArgsOwned { extra: Some(payload_args), args: HashMap::new() };
+    let args = args
+        .to_args_from_runnable(
+            &db,
+            &w_id,
+            RunnableId::from_flow_path(&flow_path.0),
+            run_query.skip_preprocessor,
+        )
+        .await?;
 
     run_wait_result_flow_by_path_internal(db, run_query, flow_path, authed, user_db, args, w_id)
         .await
@@ -4495,6 +4507,7 @@ pub async fn run_wait_result_script_by_path_internal(
     let mut tx = user_db.clone().begin(&authed).await?;
     let (job_payload, tag, delete_after_use, timeout, on_behalf_of) =
         script_path_to_payload(script_path, &mut *tx, &w_id, run_query.skip_preprocessor).await?;
+    drop(tx);
 
     let tag = run_query.tag.clone().or(tag);
     check_tag_available_for_workspace(&w_id, &tag, &authed).await?;
@@ -4715,6 +4728,7 @@ pub async fn run_wait_result_flow_by_path_internal(
         edited_by,
         version,
     } = get_latest_flow_version_info_for_path(&mut *tx, &w_id, &flow_path, true).await?;
+    drop(tx);
 
     let tag = run_query.tag.clone().or(tag);
     check_tag_available_for_workspace(&w_id, &tag, &authed).await?;

backend/windmill-api/src/lib.rs

@@ -85,13 +85,14 @@ mod http_trigger_args;
 #[cfg(feature = "http_trigger")]
 mod http_trigger_auth;
 #[cfg(feature = "http_trigger")]
-mod http_triggers;
+pub mod http_triggers;
 mod indexer_ee;
 mod inputs;
 mod integration;
 #[cfg(feature = "postgres_trigger")]
 mod postgres_triggers;
 
+mod approvals;
 #[cfg(feature = "enterprise")]
 mod apps_ee;
 #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
@@ -124,6 +125,7 @@ mod slack_approvals;
 mod smtp_server_ee;
 #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
 mod sqs_triggers_ee;
+mod teams_approvals_ee;
 mod trigger_helpers;
 
 mod static_assets;
@@ -404,6 +406,12 @@ pub async fn run_server(
         Router::new()
     };
 
+    #[cfg(feature = "http_trigger")]
+    {
+        let http_killpill_rx = killpill_rx.resubscribe();
+        http_triggers::refresh_routers_loop(&db, http_killpill_rx).await;
+    }
+
     let postgres_triggers_service = {
         #[cfg(feature = "postgres_trigger")]
         {
@@ -650,6 +658,10 @@ pub async fn run_server(
                     "/w/:workspace_id/jobs/slack_approval/:job_id",
                     get(slack_approvals::request_slack_approval),
                 )
+                .route(
+                    "/w/:workspace_id/jobs/teams_approval/:job_id",
+                    get(teams_approvals_ee::request_teams_approval),
+                )
                 .nest("/w/:workspace_id/github_app", {
                     #[cfg(feature = "enterprise")]
                     {

backend/windmill-api/src/slack_approvals.rs

@@ -3,28 +3,21 @@ use axum::{
     Extension,
 };
 use hyper::StatusCode;
-use serde::{Deserialize, Serialize};
-use serde_json::value::{RawValue, Value};
-
-use sqlx::types::Uuid;
-use std::{collections::HashMap, str::FromStr};
-
-use regex::Regex;
 use reqwest::Client;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use sqlx::types::Uuid;
+use std::collections::HashMap;
+use windmill_common::error::Error;
+use windmill_common::variables::get_secret_value_as_admin;
 
+use crate::approvals::{
+    extract_w_id_from_resume_url, handle_resume_action, ApprovalFormDetails, FieldType,
+    MessageFormat, QueryDefaultArgsJson, QueryDynamicEnumJson, QueryFlowStepId, QueryMessage,
+    ResumeFormField, ResumeSchema,
+};
 use crate::db::{ApiAuthed, DB};
-use crate::jobs::{
-    cancel_suspended_job, get_resume_urls_internal, resume_suspended_job, QueryApprover,
-    QueryOrBody, ResumeUrls,
-};
-
-use windmill_common::{
-    cache,
-    error::{self, Error},
-    jobs::JobKind,
-    scripts::ScriptHash,
-    variables::get_secret_value_as_admin,
-};
+use crate::jobs::{QueryApprover, ResumeUrls};
 
 #[derive(Deserialize, Debug)]
 pub struct SlackFormData {
@@ -91,53 +84,6 @@ struct SelectedOption {
     value: String,
 }
 
-#[derive(Debug, Deserialize, Serialize)]
-struct ResumeSchema {
-    schema: Schema,
-}
-
-#[derive(Debug, Deserialize)]
-struct ResumeFormRow {
-    resume_form: Option<serde_json::Value>,
-    hide_cancel: Option<bool>,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
-struct Schema {
-    order: Vec<String>,
-    required: Vec<String>,
-    properties: HashMap<String, ResumeFormField>,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
-#[serde(rename_all = "lowercase")]
-enum FieldType {
-    Boolean,
-    String,
-    Number,
-    Integer,
-    #[serde(other)]
-    Unknown,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
-struct ResumeFormField {
-    r#type: FieldType,
-    format: Option<String>,
-    default: Option<serde_json::Value>,
-    description: Option<String>,
-    title: Option<String>,
-    r#enum: Option<Vec<String>>,
-    #[serde(rename = "enumLabels")]
-    enum_labels: Option<HashMap<String, String>>,
-    nullable: Option<bool>,
-}
-
-#[derive(Deserialize)]
-pub struct QueryMessage {
-    message: Option<String>,
-}
-
 #[derive(Deserialize)]
 pub struct QueryResourcePath {
     slack_resource_path: String,
@@ -148,21 +94,6 @@ pub struct QueryChannelId {
     channel_id: String,
 }
 
-#[derive(Deserialize)]
-pub struct QueryFlowStepId {
-    flow_step_id: String,
-}
-
-#[derive(Deserialize, Debug)]
-pub struct QueryDefaultArgsJson {
-    default_args_json: Option<serde_json::Value>,
-}
-
-#[derive(Deserialize, Debug)]
-pub struct QueryDynamicEnumJson {
-    dynamic_enums_json: Option<serde_json::Value>,
-}
-
 #[derive(Deserialize, Debug)]
 struct ModalActionValue {
     w_id: String,
@@ -362,73 +293,21 @@ async fn handle_submission(
         return Ok(());
     }
 
-    // Use regex to extract information from private_metadata
-    let re = Regex::new(r"/api/w/(?P<w_id>[^/]+)/jobs_u/(?P<action>resume|cancel)/(?P<job_id>[^/]+)/(?P<resume_id>[^/]+)/(?P<secret>[a-fA-F0-9]+)(?:\?approver=(?P<approver>[^&]+))?").unwrap();
-    let captures = re.captures(resume_url.as_str()).ok_or_else(|| {
-        tracing::error!("Resume URL does not match the pattern.");
-        Error::BadRequest("Invalid URL format.".to_string())
-    })?;
+    // Use the common handler to process the resume/cancel action
+    handle_resume_action(authed, db.clone(), &resume_url, state_json, action).await?;
 
-    let (w_id, job_id, resume_id, secret, approver) = (
-        captures.name("w_id").map_or("", |m| m.as_str()),
-        captures.name("job_id").map_or("", |m| m.as_str()),
-        captures.name("resume_id").map_or("", |m| m.as_str()),
-        captures.name("secret").map_or("", |m| m.as_str()),
-        captures.name("approver").map(|m| m.as_str().to_string()),
-    );
-
-    let approver = QueryApprover { approver: approver };
-
-    // Convert job_id and resume_id to appropriate types
-    let job_uuid = Uuid::from_str(job_id)
-        .map_err(|_| Error::BadRequest("Invalid job ID format.".to_string()))?;
-
-    let resume_id_parsed = resume_id
-        .parse::<u32>()
-        .map_err(|_| Error::BadRequest("Invalid resume ID format.".to_string()))?;
-
-    // Call the appropriate function based on the action
-    let res = if action == "resume" {
-        resume_suspended_job(
-            authed,
-            Extension(db.clone()),
-            Path((
-                w_id.to_string(),
-                job_uuid,
-                resume_id_parsed,
-                secret.to_string(),
-            )),
-            Query(approver),
-            QueryOrBody(Some(state_json)),
-        )
-        .await
-    } else {
-        cancel_suspended_job(
-            authed,
-            Extension(db.clone()),
-            Path((
-                w_id.to_string(),
-                job_uuid,
-                resume_id_parsed,
-                secret.to_string(),
-            )),
-            Query(approver),
-            QueryOrBody(Some(state_json)),
-        )
-        .await
-    };
-    tracing::debug!("Resume job action result: {:#?}", res);
-    let slack_token = get_slack_token(&db, &resource_path, &w_id).await?;
+    let w_id = extract_w_id_from_resume_url(&resume_url)?;
+    let slack_token = get_slack_token(&db, &resource_path, w_id).await?;
     update_original_slack_message(action, slack_token, container).await?;
     Ok(())
 }
 
 async fn transform_schemas(
     text: &str,
-    properties: Option<&HashMap<String, ResumeFormField>>,
+    properties: Option<HashMap<String, ResumeFormField>>,
     urls: &ResumeUrls,
-    order: Option<&Vec<String>>,
-    required: Option<&Vec<String>>,
+    order: Option<Vec<String>>,
+    required: Option<Vec<String>>,
     default_args_json: Option<&serde_json::Value>,
     dynamic_enums_json: Option<&serde_json::Value>,
 ) -> Result<serde_json::Value, Error> {
@@ -443,16 +322,16 @@ async fn transform_schemas(
     })];
 
     if let Some(properties) = properties {
-        for key in order.unwrap() {
-            if let Some(schema) = properties.get(key) {
-                let is_required = required.unwrap().contains(key);
+        for key in order.unwrap_or_default() {
+            if let Some(schema) = properties.get(&key) {
+                let is_required = required.as_ref().map_or(false, |r| r.contains(&key));
 
-                let default_value = default_args_json.and_then(|json| json.get(key).cloned());
+                let default_value = default_args_json.and_then(|json| json.get(&key).cloned());
                 let dynamic_enums_value =
-                    dynamic_enums_json.and_then(|json| json.get(key).cloned());
+                    dynamic_enums_json.and_then(|json| json.get(&key).cloned());
 
                 let input_block = create_input_block(
-                    key,
+                    &key,
                     schema,
                     is_required,
                     default_value,
@@ -947,155 +826,59 @@ async fn get_modal_blocks(
     default_args_json: Option<&serde_json::Value>,
     dynamic_enums_json: Option<&serde_json::Value>,
 ) -> Result<axum::Json<serde_json::Value>, Error> {
-    let res = get_resume_urls_internal(
-        axum::Extension(db.clone()),
-        Path((w_id.to_string(), job_id, resume_id)),
-        Query(QueryApprover { approver: approver.map(|a| a.to_string()) }),
+    let approval_details = crate::approvals::get_approval_form_details(
+        db,
+        w_id,
+        job_id,
+        flow_step_id,
+        resume_id,
+        approver,
+        message,
+        MessageFormat::Slack,
     )
     .await?;
 
-    let urls = res.0;
+    let ApprovalFormDetails { message_str, urls, schema } = approval_details;
 
-    tracing::debug!("Job ID: {:?}", job_id);
-
-    let (job_kind, script_hash, raw_flow, parent_job_id, created_at, created_by, script_path, args) = sqlx::query!(
-        "SELECT
-            v2_as_queue.job_kind AS \"job_kind!: JobKind\",
-            v2_as_queue.script_hash AS \"script_hash: ScriptHash\",
-            v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json<Box<RawValue>>\",
-            v2_as_completed_job.parent_job AS \"parent_job: Uuid\",
-            v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\",
-            v2_as_completed_job.created_by AS \"created_by!\",
-            v2_as_queue.script_path,
-            v2_as_queue.args AS \"args: sqlx::types::Json<Box<RawValue>>\"
-        FROM v2_as_queue
-        JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id
-        WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2
-        LIMIT 1",
-        job_id,
-        &w_id
+    // Get the card content
+    let card_content = transform_schemas(
+        &message_str,
+        schema
+            .as_ref()
+            .and_then(|s| s.resume_form.as_ref())
+            .map(|f| {
+                let inner_schema: ResumeSchema = serde_json::from_value(f.clone()).unwrap();
+                inner_schema.schema.properties
+            }),
+        &urls,
+        schema
+            .as_ref()
+            .and_then(|s| s.resume_form.as_ref())
+            .map(|f| {
+                let inner_schema: ResumeSchema = serde_json::from_value(f.clone()).unwrap();
+                inner_schema.schema.order
+            }),
+        schema
+            .as_ref()
+            .and_then(|s| s.resume_form.as_ref())
+            .map(|f| {
+                let inner_schema: ResumeSchema = serde_json::from_value(f.clone()).unwrap();
+                inner_schema.schema.required
+            }),
+        default_args_json,
+        dynamic_enums_json,
     )
-    .fetch_optional(&db)
-    .await
-    .map_err(|e| error::Error::BadRequest(e.to_string()))?
-    .ok_or_else(|| error::Error::BadRequest("This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string()))
-    .map(|r| (r.job_kind, r.script_hash, r.raw_flow, r.parent_job, r.created_at, r.created_by, r.script_path, r.args))?;
+    .await?;
 
-    let flow_data = match cache::job::fetch_flow(&db, job_kind, script_hash).await {
-        Ok(data) => data,
-        Err(_) => {
-            if let Some(parent_job_id) = parent_job_id.as_ref() {
-                cache::job::fetch_preview_flow(&db, parent_job_id, raw_flow).await?
-            } else {
-                return Err(error::Error::BadRequest(
-                    "This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string(),
-                ));
-            }
-        }
-    };
-
-    let flow_value = &flow_data.flow;
-    let flow_step_id = flow_step_id.unwrap_or("");
-    let module = flow_value.modules.iter().find(|m| m.id == flow_step_id);
-
-    tracing::debug!("Module: {:#?}", module);
-
-    let schema = module.and_then(|module| {
-        module.suspend.as_ref().map(|suspend| ResumeFormRow {
-            resume_form: suspend.resume_form.clone(),
-            hide_cancel: suspend.hide_cancel,
-        })
-    });
-
-    let args_str = args.map_or("None".to_string(), |a| a.get().to_string());
-    let parent_job_id_str = parent_job_id.map_or("None".to_string(), |id| id.to_string());
-    let script_path_str = script_path.as_deref().unwrap_or("None");
-
-    let created_at_formatted = created_at.format("%Y-%m-%d %H:%M:%S").to_string();
-
-    let mut message_str = format!(
-        "A workflow has been suspended and is waiting for approval:\n\n\
-        *Created by*: {created_by}\n\
-        *Created at*: {created_at_formatted}\n\
-        *Script path*: {script_path_str}\n\
-        *Args*: {args_str}\n\
-        *Flow ID*: {parent_job_id_str}\n\n"
-    );
-
-    // Append custom message if provided
-    if let Some(msg) = message {
-        message_str.push_str(msg);
-    }
-
-    tracing::debug!("Schema: {:#?}", schema);
-
-    if let Some(resume_schema) = schema {
-        let hide_cancel = resume_schema.hide_cancel.unwrap_or(false);
-
-        // if hide cancel is false add note to message
-        if !hide_cancel {
-            message_str.push_str("\n\n*NOTE*: closing this modal will cancel the workflow.\n\n");
-        }
-
-        // Convert message_str back to &str when needed
-        let message_str_ref: &str = &message_str;
-
-        if let Some(schema_obj) = resume_schema.resume_form {
-            let inner_schema: ResumeSchema =
-                serde_json::from_value(schema_obj.clone()).map_err(|e| {
-                    tracing::error!("Failed to deserialize form schema: {:?}", e);
-                    Error::BadRequest(
-                        "Failed to deserialize resume form schema! Unsupported form field used."
-                            .to_string(),
-                    )
-                })?;
-
-            let blocks = transform_schemas(
-                message_str_ref,
-                Some(&inner_schema.schema.properties),
-                &urls,
-                Some(&inner_schema.schema.order),
-                Some(&inner_schema.schema.required),
-                default_args_json,
-                dynamic_enums_json,
-            )
-            .await?;
-
-            tracing::debug!("Slack Blocks: {:#?}", blocks);
-            return Ok(axum::Json(construct_payload(
-                blocks,
-                hide_cancel,
-                trigger_id,
-                &urls.resume,
-                resource_path,
-                container,
-            )));
-        } else {
-            tracing::debug!("No suspend form found!");
-            let blocks = transform_schemas(
-                message_str_ref,
-                None,
-                &urls,
-                None,
-                None,
-                default_args_json,
-                dynamic_enums_json,
-            )
-            .await?;
-            return Ok(axum::Json(construct_payload(
-                blocks,
-                hide_cancel,
-                trigger_id,
-                &urls.resume,
-                resource_path,
-                container,
-            )));
-        }
-    } else {
-        Err(Error::BadRequest(
-            "No approval form schema found.".to_string(),
-        ))
-    }
+    tracing::debug!("Slack Blocks: {:#?}", card_content);
+    Ok(axum::Json(construct_payload(
+        card_content,
+        schema.as_ref().and_then(|s| s.hide_cancel).unwrap_or(false),
+        trigger_id,
+        &urls.resume,
+        resource_path,
+        container,
+    )))
 }
 
 fn construct_payload(

backend/windmill-api/src/teams_approvals_ee.rs

@@ -0,0 +1,7 @@
+use hyper::StatusCode;
+
+use windmill_common::error::Error;
+
+pub async fn request_teams_approval() -> Result<StatusCode, Error> {
+    Err(Error::InternalErr("enterprise feature only".to_string()))
+}

backend/windmill-api/src/trigger_helpers.rs

@@ -4,6 +4,7 @@ use serde_json::value::RawValue;
 use std::collections::HashMap;
 use windmill_common::{
     error::Result,
+    flows::FlowModuleValue,
     get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path,
     scripts::{ScriptHash, ScriptLang},
     worker::to_raw_value,
@@ -38,12 +39,6 @@ struct ScriptInfo {
     schema: Option<sqlx::types::Json<PartialSchema>>,
 }
 
-struct FlowInfo {
-    has_preprocessor: Option<bool>,
-    is_v1_preprocessor: Option<bool>,
-    schema: Option<sqlx::types::Json<PartialSchema>>,
-}
-
 #[derive(Debug, Deserialize)]
 struct PropertyDefinition {
     r#type: Option<String>,
@@ -104,9 +99,8 @@ async fn get_script_info(
         .await
 }
 
-fn runnable_format_from_schema(
+fn runnable_format_from_schema_without_preprocessor(
     trigger_kind: &TriggerKind,
-    has_preprocessor: bool,
     schema: Option<sqlx::types::Json<PartialSchema>>,
 ) -> RunnableFormat {
     match trigger_kind {
@@ -119,7 +113,7 @@ fn runnable_format_from_schema(
                 })
             }) =>
         {
-            RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor }
+            RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false }
         }
         TriggerKind::Kafka | TriggerKind::Nats
             if schema.as_ref().is_some_and(|schema| {
@@ -129,18 +123,46 @@ fn runnable_format_from_schema(
                     .is_some_and(|properties| properties.keys().any(|key| key == "msg"))
             }) =>
         {
-            RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor }
+            RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false }
         }
-        _ => RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor },
+        _ => RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: false },
     }
 }
+
+fn runnable_format_from_preprocessor_args(
+    args: Option<Vec<windmill_parser::Arg>>,
+) -> RunnableFormat {
+    if let Some(args) = args {
+        if args.iter().any(|arg| arg.name == "wm_trigger")
+            || (args.len() > 0 && args.iter().all(|arg| arg.name != "event"))
+        {
+            RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true }
+        } else {
+            RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true }
+        }
+    } else {
+        RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true }
+    }
+}
+
+enum PreprocessorInfo {
+    Preprocessor { content: String, language: ScriptLang },
+    NoPreprocessor { schema: Option<sqlx::types::Json<PartialSchema>> },
+}
+
+#[derive(Debug, Deserialize)]
+struct FlowInfo {
+    preprocessor_module: Option<sqlx::types::Json<FlowModuleValue>>,
+    schema: Option<sqlx::types::Json<PartialSchema>>,
+}
+
 pub async fn get_runnable_format(
     runnable_id: RunnableId,
     workspace_id: &str,
     db: &DB,
     trigger_kind: &TriggerKind,
 ) -> Result<RunnableFormat> {
-    match runnable_id {
+    let (key, preprocessor_info) = match runnable_id {
         RunnableId::FlowPath(path) => {
             let FlowVersionInfo { version, .. } =
                 get_latest_flow_version_info_for_path(db, workspace_id, &path, true).await?;
@@ -157,11 +179,10 @@ pub async fn get_runnable_format(
             let flow_info = sqlx::query_as!(
                 FlowInfo,
                 "SELECT
-                    value->'preprocessor_module' IS NOT NULL as has_preprocessor,
-                    value->'preprocessor_module'->'value'->'input_transforms'->'wm_trigger' IS NOT NULL as is_v1_preprocessor,
+                    value->'preprocessor_module'->'value' as \"preprocessor_module: _\",
                     schema as \"schema: _\"
                 FROM flow 
-                WHERE workspace_id = $1 
+                WHERE workspace_id = $1
                     AND path = $2",
                 workspace_id,
                 path
@@ -169,18 +190,40 @@ pub async fn get_runnable_format(
             .fetch_one(db)
             .await?;
 
-            let has_preprocessor = flow_info.has_preprocessor.unwrap_or(false);
-            let is_v1_preprocessor = flow_info.is_v1_preprocessor.unwrap_or(false);
-
-            let runnable_format = if has_preprocessor && is_v1_preprocessor {
-                RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true }
+            if let Some(preprocessor_module) = flow_info.preprocessor_module {
+                match preprocessor_module.0 {
+                    FlowModuleValue::RawScript { content, language, .. } => {
+                        (key, PreprocessorInfo::Preprocessor { content, language })
+                    }
+                    FlowModuleValue::Script { path, hash, .. } => {
+                        let hash = if let Some(hash) = hash {
+                            hash.0
+                        } else {
+                            let script_hash =
+                                get_latest_deployed_hash_for_path(db, workspace_id, &path).await?;
+                            script_hash.hash
+                        };
+                        let script_info = get_script_info(db, workspace_id, hash).await?;
+                        (
+                            key,
+                            PreprocessorInfo::Preprocessor {
+                                content: script_info.content,
+                                language: script_info.language,
+                            },
+                        )
+                    }
+                    _ => {
+                        return Err(windmill_common::error::Error::internal_err(
+                            "Unsupported preprocessor module".to_string(),
+                        ));
+                    }
+                }
             } else {
-                runnable_format_from_schema(trigger_kind, has_preprocessor, flow_info.schema)
-            };
-
-            RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format);
-
-            Ok(runnable_format)
+                (
+                    key,
+                    PreprocessorInfo::NoPreprocessor { schema: flow_info.schema },
+                )
+            }
         }
         RunnableId::ScriptId(script_id) => {
             let hash = script_id.get_script_hash(workspace_id, db).await?;
@@ -194,47 +237,59 @@ pub async fn get_runnable_format(
 
             let script_info = get_script_info(db, workspace_id, hash).await?;
 
-            let has_preprocessor = script_info.has_preprocessor.unwrap_or(false);
-
-            let runnable_format = if has_preprocessor {
-                let args = match script_info.language {
-                    ScriptLang::Bun
-                    | ScriptLang::Bunnative
-                    | ScriptLang::Deno
-                    | ScriptLang::Nativets => {
-                        let args = windmill_parser_ts::parse_deno_signature(
-                            &script_info.content,
-                            true,
-                            false,
-                            Some("preprocessor".to_string()),
-                        )?;
-                        Some(args.args)
-                    }
-                    ScriptLang::Python3 => {
-                        let args = windmill_parser_py::parse_python_signature(
-                            &script_info.content,
-                            Some("preprocessor".to_string()),
-                            false,
-                        )?;
-                        Some(args.args)
-                    }
-                    _ => None,
-                };
-
-                if args.is_some_and(|args| args.iter().any(|arg| arg.name == "wm_trigger")) {
-                    RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true }
-                } else {
-                    runnable_format_from_schema(trigger_kind, has_preprocessor, script_info.schema)
-                }
+            if script_info.has_preprocessor.unwrap_or(false) {
+                (
+                    key,
+                    PreprocessorInfo::Preprocessor {
+                        content: script_info.content,
+                        language: script_info.language,
+                    },
+                )
             } else {
-                runnable_format_from_schema(trigger_kind, has_preprocessor, script_info.schema)
+                (
+                    key,
+                    PreprocessorInfo::NoPreprocessor { schema: script_info.schema },
+                )
+            }
+        }
+    };
+
+    let runnable_format = match preprocessor_info {
+        PreprocessorInfo::Preprocessor { content, language } => {
+            let args = match language {
+                ScriptLang::Bun
+                | ScriptLang::Bunnative
+                | ScriptLang::Deno
+                | ScriptLang::Nativets => {
+                    let args = windmill_parser_ts::parse_deno_signature(
+                        &content,
+                        true,
+                        false,
+                        Some("preprocessor".to_string()),
+                    )?;
+                    Some(args.args)
+                }
+                ScriptLang::Python3 => {
+                    let args = windmill_parser_py::parse_python_signature(
+                        &content,
+                        Some("preprocessor".to_string()),
+                        false,
+                    )?;
+                    Some(args.args)
+                }
+                _ => None,
             };
 
-            RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format);
-
-            Ok(runnable_format)
+            runnable_format_from_preprocessor_args(args)
         }
-    }
+        PreprocessorInfo::NoPreprocessor { schema } => {
+            runnable_format_from_schema_without_preprocessor(trigger_kind, schema)
+        }
+    };
+
+    RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format);
+
+    Ok(runnable_format)
 }
 
 #[allow(dead_code)]

backend/windmill-api/src/users.rs

@@ -8,6 +8,8 @@
 
 #![allow(non_snake_case)]
 
+use quick_cache::sync::Cache;
+
 use std::sync::atomic::AtomicBool;
 use std::sync::Arc;
 use std::time::Duration;
@@ -20,7 +22,8 @@ use crate::utils::{
     generate_instance_wide_unique_username, get_instance_username_or_create_pending,
 };
 use crate::{
-    db::DB, utils::require_super_admin, webhook_util::WebhookShared, COOKIE_DOMAIN, IS_SECURE,
+    auth::ExpiringAuthCache, db::DB, utils::require_super_admin, webhook_util::WebhookShared,
+    COOKIE_DOMAIN, IS_SECURE,
 };
 use argon2::{Argon2, PasswordHash, PasswordVerifier};
 use axum::{
@@ -214,6 +217,10 @@ pub async fn fetch_api_authed(
     fetch_api_authed_from_permissioned_as(permissioned_as, email, w_id, db, username_override).await
 }
 
+lazy_static::lazy_static! {
+    static ref API_AUTHED_CACHE: Cache<(String,String,String), ExpiringAuthCache> = Cache::new(300);
+}
+
 #[allow(unused)]
 pub async fn fetch_api_authed_from_permissioned_as(
     permissioned_as: String,
@@ -222,18 +229,43 @@ pub async fn fetch_api_authed_from_permissioned_as(
     db: &DB,
     username_override: Option<String>,
 ) -> error::Result<ApiAuthed> {
-    let authed =
-        fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?;
-    Ok(ApiAuthed {
-        username: authed.username,
-        email: email,
-        is_admin: authed.is_admin,
-        is_operator: authed.is_operator,
-        groups: authed.groups,
-        folders: authed.folders,
-        scopes: authed.scopes,
-        username_override: username_override,
-    })
+    let key = (w_id.to_string(), permissioned_as.clone(), email.clone());
+
+    let mut api_authed = match API_AUTHED_CACHE.get(&key) {
+        Some(expiring_authed) if expiring_authed.expiry > chrono::Utc::now() => {
+            tracing::debug!("API authed cache hit for user {}", email);
+            expiring_authed.authed
+        }
+        _ => {
+            tracing::debug!("API authed cache miss for user {}", email);
+            let authed =
+                fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?;
+
+            let api_authed = ApiAuthed {
+                username: authed.username,
+                email: email,
+                is_admin: authed.is_admin,
+                is_operator: authed.is_operator,
+                groups: authed.groups,
+                folders: authed.folders,
+                scopes: authed.scopes,
+                username_override: None,
+            };
+
+            API_AUTHED_CACHE.insert(
+                key,
+                ExpiringAuthCache {
+                    authed: api_authed.clone(),
+                    expiry: chrono::Utc::now() + chrono::Duration::try_seconds(120).unwrap(),
+                },
+            );
+
+            api_authed
+        }
+    };
+
+    api_authed.username_override = username_override;
+    Ok(api_authed)
 }
 
 #[derive(FromRow, Serialize)]

backend/windmill-api/src/utils.rs

@@ -5,7 +5,6 @@
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
-
 use axum::{body::Body, response::Response};
 use regex::Regex;
 use serde::{Deserialize, Deserializer};
@@ -415,3 +414,10 @@ pub async fn acknowledge_all_critical_alerts(
     );
     Ok("All unacknowledged critical alerts acknowledged".to_string())
 }
+
+#[cfg(feature = "http_trigger")]
+#[derive(Clone)]
+pub struct ExpiringCacheEntry<T> {
+    pub value: T,
+    pub expiry: std::time::Instant,
+}

backend/windmill-api/src/workspaces.rs

@@ -443,10 +443,12 @@ async fn get_settings(
         "SELECT workspace_id, slack_team_id, teams_team_id, teams_team_name, slack_name, slack_command_script, teams_command_script, slack_email, auto_invite_domain, auto_invite_operator, auto_add, customer_id, plan, webhook, deploy_to, ai_config, error_handler, error_handler_extra_args, error_handler_muted_on_cancel, large_file_storage, git_sync, deploy_ui, default_app, default_scripts, mute_critical_alerts, color, operator_settings, git_app_installations FROM workspace_settings WHERE workspace_id = $1",
         &w_id
     )
-    .fetch_one(&mut *tx)
+    .fetch_optional(&mut *tx)
     .await
     .map_err(|e| Error::internal_err(format!("getting settings: {e:#}")))?;
     tx.commit().await?;
+    let settings = not_found_if_none(settings, "workspace settings", &w_id)?;
+
     Ok(Json(settings))
 }
 

backend/windmill-common/Cargo.toml

@@ -12,7 +12,7 @@ tantivy = []
 prometheus = ["dep:prometheus"]
 loki = ["dep:tracing-loki"]
 benchmark = []
-parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts"]
+parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts", "dep:datafusion"]
 aws_auth = ["dep:aws-sdk-sts", "dep:aws-config"]
 otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk", 
     "dep:opentelemetry", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic"]
@@ -44,6 +44,9 @@ tracing = { workspace = true }
 axum = { workspace = true }
 hyper = { workspace = true }
 tokio = { workspace = true }
+tokio-stream.workspace = true
+tokio-util.workspace = true
+datafusion = { workspace = true, optional = true}
 reqwest = { workspace = true }
 tracing-subscriber = { workspace = true }
 lazy_static.workspace = true
@@ -67,6 +70,7 @@ async-stream.workspace = true
 const_format.workspace = true
 crc.workspace = true
 windmill-macros.workspace = true
+windmill-parser-sql.workspace = true
 jsonwebtoken.workspace = true
 backon.workspace = true
 

backend/windmill-common/src/s3_helpers.rs

@@ -16,10 +16,35 @@ use object_store::{aws::AmazonS3Builder, ClientOptions};
 use reqwest::header::HeaderMap;
 use serde::{Deserialize, Serialize};
 #[cfg(feature = "parquet")]
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 #[cfg(feature = "parquet")]
 use tokio::sync::RwLock;
 
+#[cfg(feature = "parquet")]
+use crate::error::to_anyhow;
+#[cfg(feature = "parquet")]
+use crate::utils::rd_string;
+#[cfg(feature = "parquet")]
+use bytes::Bytes;
+#[cfg(feature = "parquet")]
+use datafusion::arrow::array::{RecordBatch, RecordBatchWriter};
+#[cfg(feature = "parquet")]
+use datafusion::arrow::error::ArrowError;
+#[cfg(feature = "parquet")]
+use datafusion::arrow::json::writer::JsonArray;
+#[cfg(feature = "parquet")]
+use datafusion::arrow::{csv, json};
+#[cfg(feature = "parquet")]
+use datafusion::parquet::arrow::ArrowWriter;
+#[cfg(feature = "parquet")]
+use futures::TryStreamExt;
+#[cfg(feature = "parquet")]
+use std::io::Write;
+#[cfg(feature = "parquet")]
+use tokio::task;
+#[cfg(feature = "parquet")]
+use windmill_parser_sql::S3ModeFormat;
+
 #[cfg(feature = "parquet")]
 lazy_static::lazy_static! {
 
@@ -480,3 +505,184 @@ pub fn bundle(w_id: &str, hash: &str) -> String {
 pub fn raw_app(w_id: &str, version: &i64) -> String {
     format!("/home/rfiszel/raw_app/{}/{}", w_id, version)
 }
+
+// Originally used a Arc<Mutex<dyn RecordBatchWriter + Send>>
+// But cannot call .close() on it because it moves the value and the object is not Sized
+#[cfg(feature = "parquet")]
+enum RecordBatchWriterEnum {
+    Parquet(ArrowWriter<ChannelWriter>),
+    Csv(csv::Writer<ChannelWriter>),
+    Json(json::Writer<ChannelWriter, JsonArray>),
+}
+
+#[cfg(feature = "parquet")]
+impl RecordBatchWriter for RecordBatchWriterEnum {
+    fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
+        match self {
+            RecordBatchWriterEnum::Parquet(w) => w.write(batch).map_err(|e| e.into()),
+            RecordBatchWriterEnum::Csv(w) => w.write(batch),
+            RecordBatchWriterEnum::Json(w) => w.write(batch),
+        }
+    }
+
+    fn close(self) -> Result<(), ArrowError> {
+        match self {
+            RecordBatchWriterEnum::Parquet(w) => w.close().map_err(|e| e.into()).map(drop),
+            RecordBatchWriterEnum::Csv(w) => w.close(),
+            RecordBatchWriterEnum::Json(w) => w.close(),
+        }
+    }
+}
+
+#[cfg(feature = "parquet")]
+struct ChannelWriter {
+    sender: tokio::sync::mpsc::Sender<anyhow::Result<Bytes>>,
+}
+
+#[cfg(feature = "parquet")]
+impl Write for ChannelWriter {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        let data: Bytes = buf.to_vec().into();
+        self.sender.blocking_send(Ok(data)).map_err(|e| {
+            std::io::Error::new(
+                std::io::ErrorKind::BrokenPipe,
+                format!("Channel send error: {}", e),
+            )
+        })?;
+        Ok(buf.len())
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+#[cfg(not(feature = "parquet"))]
+pub async fn convert_json_line_stream<E: Into<anyhow::Error>>(
+    mut _stream: impl futures::TryStreamExt<Item = Result<serde_json::Value, E>> + Unpin,
+    _output_format: windmill_parser_sql::S3ModeFormat,
+) -> anyhow::Result<impl futures::TryStreamExt<Item = anyhow::Result<bytes::Bytes>>> {
+    Ok(async_stream::stream! {
+        yield Err(anyhow::anyhow!("Parquet feature is not enabled. Cannot convert JSON line stream."));
+    })
+}
+
+#[cfg(feature = "parquet")]
+pub async fn convert_json_line_stream<E: Into<anyhow::Error>>(
+    mut stream: impl TryStreamExt<Item = Result<serde_json::Value, E>> + Unpin,
+    output_format: S3ModeFormat,
+) -> anyhow::Result<impl TryStreamExt<Item = anyhow::Result<bytes::Bytes>>> {
+    const MAX_MPSC_SIZE: usize = 1000;
+
+    use datafusion::{execution::context::SessionContext, prelude::NdJsonReadOptions};
+    use futures::StreamExt;
+    use std::path::PathBuf;
+    use tokio::io::AsyncWriteExt;
+
+    let mut path = PathBuf::from(std::env::temp_dir());
+    path.push(format!("{}.json", rd_string(8)));
+    let path_str = path
+        .to_str()
+        .ok_or_else(|| anyhow::anyhow!("Invalid path"))?;
+
+    // Write the stream to a temporary file
+    let mut file: tokio::fs::File = tokio::fs::File::create(&path).await.map_err(to_anyhow)?;
+
+    while let Some(chunk) = stream.next().await {
+        match chunk {
+            Ok(chunk) => {
+                // Convert the chunk to bytes and write it to the file
+                let b: bytes::Bytes = serde_json::to_string(&chunk)?.into();
+                file.write_all(&b).await?;
+                file.write_all(b"\n").await?;
+            }
+            Err(e) => {
+                tokio::fs::remove_file(&path).await?;
+                return Err(e.into());
+            }
+        }
+    }
+
+    file.flush().await?;
+    file.sync_all().await?;
+    drop(file);
+
+    let ctx = SessionContext::new();
+    ctx.register_json(
+        "my_table",
+        path_str,
+        NdJsonReadOptions { ..Default::default() },
+    )
+    .await
+    .map_err(to_anyhow)?;
+
+    let df = ctx.sql("SELECT * FROM my_table").await.map_err(to_anyhow)?;
+    let schema = df.schema().clone().into();
+    let mut datafusion_stream = df.execute_stream().await.map_err(to_anyhow)?;
+
+    let (tx, rx) = tokio::sync::mpsc::channel(MAX_MPSC_SIZE);
+    let writer: Arc<Mutex<Option<RecordBatchWriterEnum>>> =
+        Arc::new(Mutex::new(Some(match output_format {
+            S3ModeFormat::Parquet => RecordBatchWriterEnum::Parquet(
+                ArrowWriter::try_new(ChannelWriter { sender: tx.clone() }, Arc::new(schema), None)
+                    .map_err(to_anyhow)?,
+            ),
+
+            S3ModeFormat::Csv => {
+                RecordBatchWriterEnum::Csv(csv::Writer::new(ChannelWriter { sender: tx.clone() }))
+            }
+            S3ModeFormat::Json => {
+                RecordBatchWriterEnum::Json(json::Writer::<_, JsonArray>::new(ChannelWriter {
+                    sender: tx.clone(),
+                }))
+            }
+        })));
+
+    // This spawn is so that the data is sent in the background. Else the function would deadlock
+    // when hitting the mpsc channel limit
+    task::spawn(async move {
+        while let Some(batch_result) = datafusion_stream.next().await {
+            let batch: RecordBatch = match batch_result {
+                Ok(batch) => batch,
+                Err(e) => {
+                    tracing::error!("Error in datafusion stream: {:?}", &e);
+                    match tx.send(Err(e.into())).await {
+                        Ok(_) => {}
+                        Err(e) => tracing::error!("Failed to write error to channel: {:?}", &e),
+                    }
+                    break;
+                }
+            };
+            let writer = writer.clone();
+            // Writer calls blocking_send which would crash if called from the async context
+            let write_result = task::spawn_blocking(move || {
+                // SAFETY: We await so the code is actually sequential, lock unwrap cannot panic
+                // Second unwrap is ok because we initialized the option with Some
+                writer.lock().unwrap().as_mut().unwrap().write(&batch)
+            })
+            .await;
+            match write_result {
+                Ok(Ok(_)) => {}
+                Ok(Err(e)) => {
+                    tracing::error!("Error writing batch: {:?}", &e);
+                    match tx.send(Err(e.into())).await {
+                        Ok(_) => {}
+                        Err(e) => tracing::error!("Failed to write error to channel: {:?}", &e),
+                    }
+                }
+                Err(e) => tracing::error!("Error in blocking task: {:?}", &e),
+            };
+        }
+        task::spawn_blocking(move || {
+            writer.lock().unwrap().take().unwrap().close()?;
+            drop(writer);
+            Ok::<_, anyhow::Error>(())
+        })
+        .await??;
+        drop(ctx);
+        tokio::fs::remove_file(&path).await?;
+        Ok::<_, anyhow::Error>(())
+    });
+
+    Ok(tokio_stream::wrappers::ReceiverStream::new(rx))
+}

backend/windmill-common/src/utils.rs

@@ -719,7 +719,10 @@ impl<F: Future> Future for WarnAfterFuture<F> {
         // Poll the timeout future to check if it has elapsed.
         if !*this.warned {
             if this.timeout.poll(cx).is_ready() {
-                tracing::warn!(location = this.location, "SLOW_QUERY: query to db taking longer than expected (> {} seconds). This is a sign the database is under heavy load, query is too heavy or database is undersized",
+                tracing::warn!(
+                    location = this.location,
+                    "SLOW_QUERY: query {} to db taking longer than expected (> {} seconds)",
+                    this.location,
                     this.seconds,
                 );
                 *this.warned = true;
@@ -733,7 +736,8 @@ impl<F: Future> Future for WarnAfterFuture<F> {
                     let elapsed = this.start_time.elapsed();
                     tracing::warn!(
                         location = this.location,
-                        "SLOW_QUERY: completed with total duration: {:.2?}",
+                        "SLOW_QUERY: completed query {} with total duration: {:.2?}",
+                        this.location,
                         elapsed
                     );
                 }

backend/windmill-queue/src/jobs.rs

@@ -140,6 +140,7 @@ pub struct CanceledBy {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct JobCompleted {
     pub job: Arc<MiniPulledJob>,
+    pub preprocessed_args: Option<HashMap<String, Box<RawValue>>>,
     pub result: Arc<Box<RawValue>>,
     pub result_columns: Option<Vec<String>>,
     pub mem_peak: i32,
@@ -2664,7 +2665,7 @@ async fn concurrency_key(db: &Pool<Postgres>, id: &Uuid) -> windmill_common::err
     )
 }
 
-fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> String {
+pub fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> String {
     // Save this value to avoid parsing twice
     let workspaced = x.as_str().replace("$workspace", workspace_id).to_string();
     if RE_ARG_TAG.is_match(&workspaced) {
@@ -2702,7 +2703,6 @@ fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> String {
                     .trim_matches('"')
                     .to_string()
             };
-            tracing::error!("arg_value: {}", arg_value);
             interpolated =
                 interpolated.replace(format!("$args[{}]", arg_name).as_str(), &arg_value);
         }
@@ -3887,11 +3887,13 @@ pub async fn push<'c, 'd>(
             let cache_ttl = value.cache_ttl.map(|x| x as i32);
             let custom_concurrency_key = value.concurrency_key.clone();
             let concurrency_time_window_s = value.concurrency_time_window_s;
-            let concurrent_limit = value.concurrent_limit;
+            let mut concurrent_limit = value.concurrent_limit;
 
             if !apply_preprocessor {
                 value.preprocessor_module = None;
             } else {
+                tag = None;
+                concurrent_limit = None;
                 preprocessed = Some(false);
             }
 
@@ -4173,26 +4175,7 @@ pub async fn push<'c, 'd>(
     };
 
     if concurrent_limit.is_some() {
-        let concurrency_key = custom_concurrency_key
-            .map(|x| interpolate_args(x, &args, workspace_id))
-            .unwrap_or(fullpath_with_workspace(
-                workspace_id,
-                script_path.as_ref(),
-                &job_kind,
-            ));
-        sqlx::query!(
-            "WITH inserted_concurrency_counter AS (
-                INSERT INTO concurrency_counter (concurrency_id, job_uuids) 
-                VALUES ($1, '{}'::jsonb)
-                ON CONFLICT DO NOTHING
-            )
-            INSERT INTO concurrency_key(key, job_id) VALUES ($1, $2)",
-            concurrency_key,
-            job_id,
-        )
-        .execute(&mut *tx)
-        .await
-        .map_err(|e| Error::internal_err(format!("Could not insert concurrency_key={concurrency_key} for job_id={job_id} script_path={script_path:?} workspace_id={workspace_id}: {e:#}")))?;
+        insert_concurrency_key(workspace_id, &args, &script_path, job_kind, custom_concurrency_key, &mut tx, job_id).await?;
     }
 
     let stringified_args = if *JOB_ARGS_AUDIT_LOGS {
@@ -4210,6 +4193,7 @@ pub async fn push<'c, 'd>(
         Some("preprocessor") => Some(false),
         _ => None,
     });
+    
 
     let job_authed = match authed {
         Some(authed)
@@ -4356,6 +4340,7 @@ pub async fn push<'c, 'd>(
             Json(flow_status) as Json<FlowStatus>,
         )
         .execute(&mut *tx)
+        .warn_after_seconds(1)
         .await?;
     }
 
@@ -4424,12 +4409,38 @@ pub async fn push<'c, 'd>(
             script_path.as_ref().map(|x| x.as_str()),
             Some(hm),
         )
+        .warn_after_seconds(1)
         .await?;
     }
 
     Ok((job_id, tx))
 }
 
+pub async fn insert_concurrency_key<'d, 'c>(workspace_id: &str, args: &PushArgs<'d>, script_path: &Option<String>, job_kind: JobKind, custom_concurrency_key: Option<String>, tx: &mut Transaction<'c, Postgres>, job_id: Uuid) -> Result<(), Error> {
+    let concurrency_key = custom_concurrency_key
+        .map(|x| interpolate_args(x, args, workspace_id))
+        .unwrap_or(fullpath_with_workspace(
+            workspace_id,
+            script_path.as_ref(),
+            &job_kind,
+        ));
+    sqlx::query!(
+        "WITH inserted_concurrency_counter AS (
+                INSERT INTO concurrency_counter (concurrency_id, job_uuids) 
+                VALUES ($1, '{}'::jsonb)
+                ON CONFLICT DO NOTHING
+            )
+            INSERT INTO concurrency_key(key, job_id) VALUES ($1, $2)",
+        concurrency_key,
+        job_id,
+    )
+    .execute(&mut **tx)
+    .warn_after_seconds(3)
+    .await
+    .map_err(|e| Error::internal_err(format!("Could not insert concurrency_key={concurrency_key} for job_id={job_id} script_path={script_path:?} workspace_id={workspace_id}: {e:#}")))?;
+    Ok(())
+}
+
 pub fn canceled_job_to_result(job: &MiniPulledJob) -> serde_json::Value {
     let reason = job
         .canceled_reason

backend/windmill-worker/Cargo.toml

@@ -57,6 +57,7 @@ sqlx.workspace = true
 uuid.workspace = true
 tracing.workspace = true
 tokio.workspace = true
+tokio-stream.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 futures.workspace = true
@@ -90,6 +91,7 @@ deno_tls = { workspace = true, optional = true }
 deno_permissions = { workspace = true, optional = true }
 deno_io = { workspace = true, optional = true }
 deno_error = { workspace = true, optional = true }
+async-stream.workspace = true
 
 postgres-native-tls.workspace = true
 native-tls.workspace = true

backend/windmill-worker/src/ansible_executor.rs

@@ -786,14 +786,30 @@ pub async fn handle_ansible_job(
         if let Ok(lockfile) = serde_json::from_str(s) {
             Some(lockfile)
         } else {
-            append_logs(
-                &job.id,
-                &job.workspace_id,
-                format!("WARN: lockfile could not be parsed: `{s}`"),
-                conn,
-            )
-            .await;
-            None
+            if !s.trim_start().starts_with('{') {
+                append_logs(
+                    &job.id,
+                    &job.workspace_id,
+                    format!("WARN: lockfile seems to be in an older version, roles and collections are therefore using the latest version and not the one locked at deployment. Redeploy the script to correct this"),
+                    conn,
+                )
+                .await;
+                Some(AnsibleDependencyLocks {
+                    python_lockfile: s.to_string(),
+                    git_repos: HashMap::new(),
+                    collections_and_roles: String::new(),
+                    collections_and_roles_logs: String::new(),
+                })
+            } else {
+                append_logs(
+                    &job.id,
+                    &job.workspace_id,
+                    format!("WARN: lockfile could not be parsed: {s}"),
+                    conn,
+                )
+                .await;
+                None
+            }
         }
     } else {
         None
@@ -948,7 +964,13 @@ pub async fn handle_ansible_job(
             let empty = String::new();
             let (lockfile, logs) = req_lockfiles
                 .as_ref()
-                .map(|r| (&r.collections_and_roles, &r.collections_and_roles_logs))
+                .and_then(|r| {
+                    if r.collections_and_roles.is_empty() {
+                        None
+                    } else {
+                        Some((&r.collections_and_roles, &r.collections_and_roles_logs))
+                    }
+                })
                 .unwrap_or((collections, &empty));
 
             if !logs.is_empty() {

backend/windmill-worker/src/bash_executor.rs

@@ -285,6 +285,8 @@ async fn handle_docker_job(
     occupancy_metrics: &mut OccupancyMetrics,
     killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
 ) -> Result<Box<RawValue>, Error> {
+    use crate::job_logger::append_logs_with_compaction;
+
     let client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow)?;
 
     let container_id = job_id.to_string();
@@ -313,6 +315,7 @@ async fn handle_docker_job(
     let w_id = workspace_id.to_string();
     let j_id = job_id.clone();
     let conn2 = conn.clone();
+    let worker_name2 = worker_name.to_string();
     let (tx, mut rx) = tokio::sync::broadcast::channel::<()>(1);
 
     let mut killpill_rx = killpill_rx.resubscribe();
@@ -334,7 +337,21 @@ async fn handle_docker_job(
                     log = log_stream.next() => {
                         match log {
                             Some(Ok(log)) => {
-                                append_logs(&j_id, w_id.clone(), log.to_string(), &conn2).await;
+                                match &conn2 {
+                                    Connection::Sql(db) => {
+                                        append_logs_with_compaction(
+                                            &j_id,
+                                            &w_id,
+                                            &log.to_string(),
+                                            &db,
+                                            &worker_name2,
+                                        )
+                                        .await;
+                                    }
+                                    c @ Connection::Http(_) => {
+                                        append_logs(&j_id, &w_id, &log.to_string(), &c).await;
+                                    }
+                                }
                             }
                             Some(Err(e)) => {
                                 tracing::error!("Error getting logs: {:?}", e);

backend/windmill-worker/src/bigquery_executor.rs

@@ -1,20 +1,24 @@
 use std::collections::HashMap;
 
 use futures::future::BoxFuture;
-use futures::FutureExt;
+use futures::{FutureExt, StreamExt};
 use reqwest::Client;
 use serde_json::{json, value::RawValue, Value};
 use windmill_common::error::to_anyhow;
+use windmill_common::s3_helpers::convert_json_line_stream;
 use windmill_common::worker::Connection;
 use windmill_common::{error::Error, worker::to_raw_value};
 use windmill_parser_sql::{
-    parse_bigquery_sig, parse_db_resource, parse_sql_blocks, parse_sql_statement_named_params,
+    parse_bigquery_sig, parse_db_resource, parse_s3_mode, parse_sql_blocks,
+    parse_sql_statement_named_params,
 };
 use windmill_queue::CanceledBy;
 
 use serde::Deserialize;
 
-use crate::common::{build_http_client, OccupancyMetrics};
+use crate::common::{
+    build_http_client, s3_mode_args_to_worker_data, OccupancyMetrics, S3ModeWorkerData,
+};
 use crate::handle_child::run_future_with_polling_update_job_poller;
 use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
 use crate::{
@@ -31,6 +35,16 @@ struct BigqueryResponse {
     totalRows: Option<Value>,
     schema: Option<BigqueryResponseSchema>,
     jobComplete: bool,
+    pageToken: Option<String>,
+    jobReference: Option<BigQueryResponseJobReference>,
+}
+
+#[allow(non_snake_case)]
+#[derive(Deserialize, Clone)]
+struct BigQueryResponseJobReference {
+    jobId: String,
+    projectId: String,
+    location: Option<String>,
 }
 
 #[derive(Deserialize)]
@@ -74,6 +88,7 @@ fn do_bigquery_inner<'a>(
     column_order: Option<&'a mut Option<Vec<String>>>,
     skip_collect: bool,
     http_client: &'a Client,
+    s3: Option<S3ModeWorkerData>,
 ) -> windmill_common::error::Result<BoxFuture<'a, windmill_common::error::Result<Box<RawValue>>>> {
     let param_names = parse_sql_statement_named_params(query, '@');
 
@@ -120,69 +135,80 @@ fn do_bigquery_inner<'a>(
                             e.to_string()
                         ))
                     })?;
+                    let rows = handle_bigquery_response(&result, &s3, column_order).await?;
 
-                    if !result.jobComplete {
-                        return Err(Error::ExecutionErr(
-                            "BigQuery API did not answer query in time".to_string(),
-                        ));
+                    if let Some(s3) = s3 {
+                        let cloned_s3 = s3.clone();
+                        let cloned_http_client = http_client.clone();
+                        let cloned_token = token.to_string();
+                        let rows_stream = async_stream::stream! {
+                            for row in rows.iter() {
+                                yield Ok::<_, windmill_common::error::Error>(row.clone());
+                            }
+                            let mut next_page_token = result.pageToken;
+                            let Some(job_reference) = result.jobReference.clone() else {
+                                return;
+                            };
+                            while let Some(ref next_page_token_value) = next_page_token {
+                                let response2 = cloned_http_client
+                                    .get(
+                                        format!("https://bigquery.googleapis.com/bigquery/v2/projects/{}/queries/{}", job_reference.projectId, job_reference.jobId),
+                                    )
+                                    .bearer_auth(cloned_token.as_str())
+                                    .query(&[
+                                        ("pageToken", next_page_token_value.as_str()),
+                                        ("maxResults", "10000"),
+                                        ("timeoutMs", timeout_ms.to_string().as_str()),
+                                        ("location", job_reference.location.as_ref().unwrap_or(&"US".to_string()).as_str()),
+                                    ])
+                                    .send()
+                                    .await
+                                    .map_err(|e| {
+                                        Error::ExecutionErr(format!("Could not send query to BigQuery API: {}", e))
+                                    })?;
+
+                                if let Err(e) = response2.error_for_status_ref() {
+                                    match response2.json::<BigqueryErrorResponse>().await {
+                                        Ok(bq_err) => {
+                                            yield Err(Error::ExecutionErr(format!(
+                                                "Error from BigQuery API: {}",
+                                                bq_err.error.message
+                                            )))
+                                            .map_err(to_anyhow)?;
+                                            return;
+                                    },
+                                        Err(_) => {
+                                            yield Err(Error::ExecutionErr(format!(
+                                                "Error from BigQuery API could not be parsed: {}",
+                                                e.to_string()
+                                            )))
+                                            .map_err(to_anyhow)?;
+                                            return;
+                                        },
+                                    }
+                                }
+
+                                let result2 = response2.json::<BigqueryResponse>().await.map_err(|e| {
+                                    Error::ExecutionErr(format!(
+                                        "BigQuery API response could not be parsed: {}",
+                                        e.to_string()
+                                    ))
+                                })?;
+                                let rows = handle_bigquery_response(&result2, &Some(cloned_s3.clone()), None).await?;
+                                for row in rows.into_iter() {
+                                    yield Ok::<_, windmill_common::error::Error>(row);
+                                }
+                                next_page_token = result2.pageToken;
+                            }
+                        };
+
+                        let stream =
+                            convert_json_line_stream(rows_stream.boxed(), s3.format).await?;
+                        s3.upload(stream.boxed()).await?;
+
+                        return Ok(to_raw_value(&s3.to_return_s3_obj()));
                     }
 
-                    if result.rows.is_none() || result.rows.as_ref().unwrap().len() == 0 {
-                        return Ok(serde_json::from_str("[]").unwrap());
-                    }
-
-                    if result.schema.is_none() {
-                        return Err(Error::ExecutionErr(
-                            "Incomplete response from BigQuery API".to_string(),
-                        ));
-                    }
-
-                    if result
-                        .totalRows
-                        .unwrap_or(json!(""))
-                        .as_str()
-                        .unwrap_or("")
-                        .parse::<i64>()
-                        .unwrap_or(0)
-                        > 10000
-                    {
-                        return Err(Error::ExecutionErr(
-                        "More than 10000 rows were requested, use LIMIT 10000 to limit the number of rows".to_string(),
-                    ));
-                    }
-
-                    if let Some(column_order) = column_order {
-                        *column_order = Some(
-                            result
-                                .schema
-                                .as_ref()
-                                .unwrap()
-                                .fields
-                                .iter()
-                                .map(|x| x.name.clone())
-                                .collect::<Vec<String>>(),
-                        );
-                    }
-
-                    let rows = result
-                        .rows
-                        .unwrap()
-                        .iter()
-                        .map(|row| {
-                            let mut row_map = serde_json::Map::new();
-                            row.f
-                                .iter()
-                                .zip(result.schema.as_ref().unwrap().fields.iter())
-                                .for_each(|(field, schema)| {
-                                    row_map.insert(
-                                        schema.name.clone(),
-                                        parse_val(&field.v, &schema.r#type, &schema),
-                                    );
-                                });
-                            Value::from(row_map)
-                        })
-                        .collect::<Vec<_>>();
-
                     Ok(to_raw_value(&rows))
                 }
             }
@@ -204,6 +230,79 @@ fn do_bigquery_inner<'a>(
     Ok(result_f.boxed())
 }
 
+async fn handle_bigquery_response<'a>(
+    result: &BigqueryResponse,
+    s3: &Option<S3ModeWorkerData>,
+    column_order: Option<&'a mut Option<Vec<String>>>,
+) -> windmill_common::error::Result<Vec<Value>> {
+    if !result.jobComplete {
+        return Err(Error::ExecutionErr(
+            "BigQuery API did not answer query in time".to_string(),
+        ));
+    }
+
+    if result.rows.is_none() || result.rows.as_ref().unwrap().len() == 0 {
+        return Ok(serde_json::from_str("[]").unwrap());
+    }
+
+    if result.schema.is_none() {
+        return Err(Error::ExecutionErr(
+            "Incomplete response from BigQuery API".to_string(),
+        ));
+    }
+
+    if s3.is_none()
+        && result
+            .totalRows
+            .as_ref()
+            .unwrap_or(&json!(""))
+            .as_str()
+            .unwrap_or("")
+            .parse::<i64>()
+            .unwrap_or(0)
+            > 10000
+    {
+        return Err(Error::ExecutionErr(
+            "More than 10000 rows were requested, use LIMIT 10000 to limit the number of rows"
+                .to_string(),
+        ));
+    }
+
+    if let Some(column_order) = column_order {
+        *column_order = Some(
+            result
+                .schema
+                .as_ref()
+                .unwrap()
+                .fields
+                .iter()
+                .map(|x| x.name.clone())
+                .collect::<Vec<String>>(),
+        );
+    }
+
+    let rows = result
+        .rows
+        .as_ref()
+        .unwrap()
+        .iter()
+        .map(|row| {
+            let mut row_map = serde_json::Map::new();
+            row.f
+                .iter()
+                .zip(result.schema.as_ref().unwrap().fields.iter())
+                .for_each(|(field, schema)| {
+                    row_map.insert(
+                        schema.name.clone(),
+                        parse_val(&field.v, &schema.r#type, &schema),
+                    );
+                });
+            Value::from(row_map)
+        })
+        .collect::<Vec<_>>();
+    Ok(rows)
+}
+
 use windmill_queue::MiniPulledJob;
 
 pub async fn do_bigquery(
@@ -220,6 +319,7 @@ pub async fn do_bigquery(
     let bigquery_args = build_args_values(job, client, conn).await?;
 
     let inline_db_res_path = parse_db_resource(&query);
+    let s3 = parse_s3_mode(&query)?.map(|s3| s3_mode_args_to_worker_data(s3, client.clone(), job));
 
     let db_arg = if let Some(inline_db_res_path) = inline_db_res_path {
         Some(
@@ -332,6 +432,7 @@ pub async fn do_bigquery(
                     None,
                     annotations.return_last_result && i < queries.len() - 1,
                     &http_client,
+                    s3.clone(),
                 )
             })
             .collect::<windmill_common::error::Result<Vec<_>>>()?;
@@ -361,6 +462,7 @@ pub async fn do_bigquery(
             Some(column_order),
             false,
             &http_client,
+            s3,
         )?
     };
 

backend/windmill-worker/src/common.rs

@@ -31,6 +31,7 @@ use windmill_common::{
 };
 
 use anyhow::{anyhow, bail, Result};
+use windmill_parser_sql::{s3_mode_extension, S3ModeArgs, S3ModeFormat};
 use windmill_queue::MiniPulledJob;
 
 use std::ops::AsyncFn;
@@ -1579,3 +1580,63 @@ pub async fn par_install_language_dependencies<'a>(
     }
     Ok(())
 }
+
+#[derive(Clone)]
+pub struct S3ModeWorkerData {
+    pub client: AuthedClient,
+    pub object_key: String,
+    pub format: S3ModeFormat,
+    pub storage: Option<String>,
+    pub workspace_id: String,
+}
+
+impl S3ModeWorkerData {
+    pub async fn upload<S>(&self, stream: S) -> error::Result<()>
+    where
+        S: futures::stream::TryStream + Send + 'static,
+        S::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
+        bytes::Bytes: From<S::Ok>,
+    {
+        self.client
+            .upload_s3_file(
+                self.workspace_id.as_str(),
+                self.object_key.clone(),
+                self.storage.clone(),
+                stream,
+            )
+            .await
+    }
+
+    pub fn to_return_s3_obj(&self) -> windmill_common::s3_helpers::S3Object {
+        windmill_common::s3_helpers::S3Object {
+            s3: self.object_key.clone(),
+            storage: self.storage.clone(),
+            ..Default::default()
+        }
+    }
+}
+
+pub fn s3_mode_args_to_worker_data(
+    s3: S3ModeArgs,
+    client: AuthedClient,
+    job: &MiniPulledJob,
+) -> S3ModeWorkerData {
+    S3ModeWorkerData {
+        client,
+        storage: s3.storage,
+        format: s3.format,
+        object_key: format!(
+            "{}/{}.{}",
+            s3.prefix.unwrap_or_else(|| format!(
+                "wmill_datalake/{}",
+                job.runnable_path
+                    .as_ref()
+                    .map(|s| s.as_str())
+                    .unwrap_or("unknown_script")
+            )),
+            job.id,
+            s3_mode_extension(s3.format)
+        ),
+        workspace_id: job.workspace_id.clone(),
+    }
+}

backend/windmill-worker/src/dedicated_worker.rs

@@ -187,14 +187,14 @@ pub async fn handle_dedicated_process(
                                 let result = Arc::new(result);
                                 append_logs(&job.id, &job.workspace_id,  logs.clone(), &db.into()).await;
                                 if line.starts_with("wm_res[success]:") {
-                                    job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap()
+                                    job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None }, true).await.unwrap()
                                 } else {
-                                    job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap()
+                                    job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None }, true).await.unwrap()
                                 }
                             },
                             Err(e) => {
                                 tracing::error!("Could not deserialize job result `{line}`: {e:?}");
-                                job_completed_tx.send_job(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap();
+                                job_completed_tx.send_job(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None }, true).await.unwrap();
                             },
                         };
                         logs = init_log.clone();

backend/windmill-worker/src/handle_child.rs

@@ -130,13 +130,13 @@ pub async fn handle_child(
     } else {
         tracing::info!("could not get child pid");
     }
-    let (set_too_many_logs, mut too_many_logs) = watch::channel::<bool>(false);
+    let (mut set_too_many_logs, mut too_many_logs) = watch::channel::<bool>(false);
     let (tx, rx) = broadcast::channel::<()>(3);
-    let mut rx2 = tx.subscribe();
+    let mut rx2: broadcast::Receiver<()> = tx.subscribe();
 
     let output = child_joined_output_stream(&mut child, job_id.clone());
 
-    let job_id = job_id.clone();
+    let job_id: Uuid = job_id.clone();
 
     /* the cancellation future is polled on by `wait_on_child` while
      * waiting for the child to exit normally */
@@ -296,147 +296,19 @@ pub async fn handle_child(
     };
 
     /* a future that reads output from the child and appends to the database */
-    let lines = async move {
-
-        let max_log_size = if *CLOUD_HOSTED {
-            MAX_RESULT_SIZE
-        } else {
-            usize::MAX
-        };
-
-        /* log_remaining is zero when output limit was reached */
-        let mut log_remaining =  if *CLOUD_HOSTED {
-            max_log_size
-        } else {
-            usize::MAX
-        };
-        let mut result = io::Result::Ok(());
-        let mut output = output.take_until(async {
-            let _ = rx2.recv().await;
-            //wait at most 50ms after end of a script for output stream to end
-            tokio::time::sleep(Duration::from_millis(50)).await;
-         }).boxed();
-        /* `do_write` resolves the task, but does not contain the Result.
-         * It's useful to know if the task completed. */
-        let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle();
-
-        let mut log_total_size: u64 = 0;
-        let pg_log_total_size = Arc::new(AtomicU32::new(0));
-
-        let mut pipe_stdout = pipe_stdout;
-
-        while let Some(line) =  output.by_ref().next().await {
-
-            let do_write_ = do_write.shared();
-
-            let delay = if start.elapsed() < Duration::from_secs(10) {
-                Duration::from_millis(500)
-            } else if start.elapsed() < Duration::from_secs(60){
-                Duration::from_millis(2500)
-            } else {
-                Duration::from_millis(5000)
-            };
-
-            let delay = if *SLOW_LOGS {
-                delay * 10
-            } else {
-                delay
-            };
-
-            let mut read_lines = stream::once(async { line })
-                .chain(output.by_ref())
-                /* after receiving a line, continue until some delay has passed
-                 * _and_ the previous database write is complete */
-                .take_until(future::join(sleep(delay), do_write_.clone()))
-                .boxed();
-
-            /* Read up until an error is encountered,
-             * handle log lines first and then the error... */
-            let mut joined = String::new();
-
-            while let Some(line) = read_lines.next().await {
-
-                match line {
-                    Ok(line) => {
-                        if line.is_empty() {
-                            continue;
-                        }
-                        append_with_limit(&mut joined, &line, &mut log_remaining);
-                        if log_remaining == 0 {
-                            tracing::info!(%job_id, "Too many logs lines for job {job_id}");
-                            let _ = set_too_many_logs.send(true);
-                            joined.push_str(&format!(
-                                "Job logs or result reached character limit of {MAX_RESULT_SIZE}; killing job."
-                            ));
-                            /* stop reading and drop our streams fairly quickly */
-                            break;
-                        }
-                    }
-                    Err(err) => {
-                        result = Err(err);
-                        break;
-                    }
-                }
-            }
-
-
-            /* Ensure the last flush completed before starting a new one.
-             *
-             * This shouldn't pause since `take_until()` reads lines until `do_write`
-             * resolves. We only stop reading lines before `take_until()` resolves if we reach
-             * EOF or a read error.  In those cases, waiting on a database query to complete is
-             * fine because we're done. */
-
-            if let Some(Ok(p)) = do_write_
-                .then(|()| write_result)
-                .await
-                .err()
-                .map(|err| err.try_into_panic())
-            {
-                panic::resume_unwind(p);
-            }
-
-
-            let joined_len = joined.len() as u64;
-            log_total_size += joined_len;
-            let compact_logs = log_total_size > LARGE_LOG_THRESHOLD_SIZE as u64;
-            if compact_logs {
-                log_total_size = 0;
-            }
-
-            let worker_name = worker.to_string();
-            let w_id2 = w_id.to_string();
-
-
-            if let Some(buf) = &mut pipe_stdout {
-                buf.push_str(&joined);
-                (do_write, write_result) = tokio::spawn(async { }).remote_handle();
-            } else {
-                (do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, conn.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle();
-            }
-
-            if let Err(err) = result {
-                tracing::error!(%job_id, %err, "error reading output for job {job_id} '{child_name}': {err}");
-                break;
-            }
-
-            if *set_too_many_logs.borrow() {
-                break;
-            }
-        }
-
-        /* drop our end of the pipe */
-        drop(output);
-
-        if let Some(Ok(p)) = do_write
-            .then(|()| write_result)
-            .await
-            .err()
-            .map(|err| err.try_into_panic())
-        {
-            panic::resume_unwind(p);
-        }
-    }.instrument(trace_span!("child_lines"));
+    let lines = write_lines(
+        output,
+        &job_id,
+        w_id,
+        worker,
+        conn,
+        &mut set_too_many_logs,
+        start,
+        pipe_stdout,
+        &mut rx2,
+        child_name,
+    )
+    .instrument(trace_span!("child_lines"));
 
     let (wait_result, _) = tokio::join!(wait_on_child, lines);
 
@@ -462,6 +334,169 @@ pub async fn handle_child(
     }
 }
 
+pub async fn write_lines(
+    output: impl stream::Stream<Item = io::Result<String>> + Send,
+    job_id: &Uuid,
+    w_id: &str,
+    worker: &str,
+    conn: &Connection,
+    set_too_many_logs: &mut watch::Sender<bool>,
+    start: Instant,
+    pipe_stdout: Option<&mut String>,
+    rx2: &mut broadcast::Receiver<()>,
+    child_name: &str,
+) {
+    let max_log_size = if *CLOUD_HOSTED {
+        MAX_RESULT_SIZE
+    } else {
+        usize::MAX
+    };
+
+    /* log_remaining is zero when output limit was reached */
+    let mut log_remaining = if *CLOUD_HOSTED {
+        max_log_size
+    } else {
+        usize::MAX
+    };
+    let mut result = io::Result::Ok(());
+    let mut output = output
+        .take_until(async {
+            let _ = rx2.recv().await;
+            //wait at most 50ms after end of a script for output stream to end
+            tokio::time::sleep(Duration::from_millis(50)).await;
+        })
+        .boxed();
+    /* `do_write` resolves the task, but does not contain the Result.
+     * It's useful to know if the task completed. */
+    let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle();
+
+    let mut log_total_size: u64 = 0;
+    let pg_log_total_size = Arc::new(AtomicU32::new(0));
+
+    let mut pipe_stdout = pipe_stdout;
+
+    while let Some(line) = output.by_ref().next().await {
+        let do_write_ = do_write.shared();
+
+        let delay = if start.elapsed() < Duration::from_secs(10) {
+            Duration::from_millis(500)
+        } else if start.elapsed() < Duration::from_secs(60) {
+            Duration::from_millis(2500)
+        } else {
+            Duration::from_millis(5000)
+        };
+
+        let delay = if *SLOW_LOGS { delay * 10 } else { delay };
+
+        let mut read_lines = stream::once(async { line })
+            .chain(output.by_ref())
+            /* after receiving a line, continue until some delay has passed
+             * _and_ the previous database write is complete */
+            .take_until(future::join(sleep(delay), do_write_.clone()))
+            .boxed();
+
+        /* Read up until an error is encountered,
+         * handle log lines first and then the error... */
+        let mut joined = String::new();
+
+        let job_id = job_id.clone();
+        while let Some(line) = read_lines.next().await {
+            match line {
+                Ok(line) => {
+                    if line.is_empty() {
+                        continue;
+                    }
+                    append_with_limit(&mut joined, &line, &mut log_remaining);
+                    if log_remaining == 0 {
+                        tracing::info!(%job_id, "Too many logs lines for job {job_id}");
+                        let _ = set_too_many_logs.send(true);
+                        joined.push_str(&format!(
+                                "Job logs or result reached character limit of {MAX_RESULT_SIZE}; killing job."
+                            ));
+                        /* stop reading and drop our streams fairly quickly */
+                        break;
+                    }
+                }
+                Err(err) => {
+                    result = Err(err);
+                    break;
+                }
+            }
+        }
+
+        /* Ensure the last flush completed before starting a new one.
+         *
+         * This shouldn't pause since `take_until()` reads lines until `do_write`
+         * resolves. We only stop reading lines before `take_until()` resolves if we reach
+         * EOF or a read error.  In those cases, waiting on a database query to complete is
+         * fine because we're done. */
+
+        if let Some(Ok(p)) = do_write_
+            .then(|()| write_result)
+            .await
+            .err()
+            .map(|err| err.try_into_panic())
+        {
+            panic::resume_unwind(p);
+        }
+
+        let joined_len = joined.len() as u64;
+        log_total_size += joined_len;
+        let compact_logs = log_total_size > LARGE_LOG_THRESHOLD_SIZE as u64;
+        if compact_logs {
+            log_total_size = 0;
+        }
+
+        let worker_name = worker.to_string();
+
+        if let Some(buf) = &mut pipe_stdout {
+            buf.push_str(&joined);
+            (do_write, write_result) = tokio::spawn(async {}).remote_handle();
+        } else {
+            let conn = conn.clone();
+            let worker_name = worker_name.to_string();
+            let w_id = w_id.to_string();
+            let job_id = job_id.clone();
+            let pg_log_total_size = pg_log_total_size.clone();
+
+            (do_write, write_result) = tokio::spawn(async move {
+                append_job_logs(
+                    &job_id,
+                    &w_id,
+                    &joined,
+                    &conn,
+                    compact_logs,
+                    pg_log_total_size,
+                    &worker_name,
+                )
+                .await;
+            })
+            .remote_handle();
+        }
+
+        if let Err(err) = result {
+            tracing::error!(%job_id, %err, "error reading output for job {job_id} '{child_name}': {err}");
+            break;
+        }
+
+        if *set_too_many_logs.borrow() {
+            break;
+        }
+    }
+
+    /* drop our end of the pipe */
+    drop(output);
+
+    if let Some(Ok(p)) = do_write
+        .then(|()| write_result)
+        .await
+        .err()
+        .map(|err| err.try_into_panic())
+    {
+        panic::resume_unwind(p);
+    }
+}
+
 pub(crate) async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
     if pid.is_none() {
         return -1;

backend/windmill-worker/src/job_logger.rs

@@ -1,8 +1,10 @@
 use regex::Regex;
 
 pub use windmill_common::jobs::LARGE_LOG_THRESHOLD_SIZE;
+use windmill_common::utils::WarnAfterExt;
 use windmill_common::worker::{Connection, CLOUD_HOSTED};
 
+use windmill_common::DB;
 use windmill_queue::append_logs;
 
 use std::sync::atomic::AtomicU32;
@@ -26,23 +28,23 @@ pub enum CompactLogs {
 }
 
 pub async fn append_job_logs(
-    job_id: Uuid,
-    w_id: String,
-    logs: String,
-    conn: Connection,
+    job_id: &Uuid,
+    w_id: &str,
+    logs: &str,
+    conn: &Connection,
     must_compact_logs: bool,
     total_size: Arc<AtomicU32>,
-    worker_name: String,
+    worker_name: &str,
 ) -> () {
     match conn {
         Connection::Sql(db) if must_compact_logs => {
             #[cfg(all(feature = "enterprise", feature = "parquet"))]
-            s3_storage(job_id, &w_id, &db, logs, total_size, &worker_name).await;
+            s3_storage(&job_id, &w_id, &db, logs, total_size, worker_name).await;
 
             #[cfg(not(all(feature = "enterprise", feature = "parquet")))]
             {
                 default_disk_log_storage(
-                    job_id,
+                    &job_id,
                     &w_id,
                     &db,
                     logs,
@@ -59,6 +61,45 @@ pub async fn append_job_logs(
     }
 }
 
+pub async fn append_logs_with_compaction(
+    job_id: &Uuid,
+    w_id: &str,
+    logs: &str,
+    db: &DB,
+    worker_name: &str,
+) {
+    let log_length = sqlx::query_scalar!(
+        "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, $1::text) RETURNING length(logs)",
+        logs,
+        job_id,
+        &w_id,
+    )
+    .fetch_one(db)
+    .warn_after_seconds(1)
+    .await;
+    match log_length {
+        Ok(length) => {
+            let len = length.unwrap_or(0);
+            let conn: Connection = db.into();
+            if len > LARGE_LOG_THRESHOLD_SIZE as i32 {
+                append_job_logs(
+                    &job_id,
+                    w_id,
+                    "",
+                    &conn,
+                    true,
+                    Arc::new(AtomicU32::new(len as u32)),
+                    worker_name,
+                )
+                .await;
+            }
+        }
+        Err(err) => {
+            tracing::error!(%job_id, %err, "error updating logs for job {job_id}: {err}");
+        }
+    }
+}
+
 lazy_static::lazy_static! {
     static ref RE_00: Regex = Regex::new('\u{00}'.to_string().as_str()).unwrap();
     pub static ref NO_LOGS_AT_ALL: bool = std::env::var("NO_LOGS_AT_ALL").ok().is_some_and(|x| x == "1" || x == "true");

backend/windmill-worker/src/job_logger_ee.rs

@@ -9,22 +9,22 @@ use crate::job_logger::CompactLogs;
 
 #[cfg(all(feature = "enterprise", feature = "parquet"))]
 pub(crate) async fn s3_storage(
-    _job_id: Uuid,
-    _w_id: &String,
+    _job_id: &Uuid,
+    _w_id: &str,
     _db: &sqlx::Pool<sqlx::Postgres>,
-    _logs: String,
+    _logs: &str,
     _total_size: Arc<AtomicU32>,
-    _worker_name: &String,
+    _worker_name: &str,
 ) {
     tracing::info!("Logs length of {_job_id} has exceeded a threshold. Implementation to store excess on s3 in not OSS");
 }
 
 #[allow(dead_code)]
 pub(crate) async fn default_disk_log_storage(
-    job_id: Uuid,
+    job_id: &Uuid,
     _w_id: &str,
     _db: &DB,
-    _nlogs: String,
+    _logs: &str,
     _total_size: Arc<AtomicU32>,
     _compact_kind: CompactLogs,
     _worker_name: &str,

backend/windmill-worker/src/mssql_executor.rs

@@ -1,5 +1,6 @@
 use base64::{engine::general_purpose, Engine as _};
 use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
+use futures::StreamExt;
 use regex::Regex;
 use serde::Deserialize;
 use serde_json::value::RawValue;
@@ -8,16 +9,17 @@ use tiberius::{AuthMethod, Client, ColumnData, Config, FromSqlOwned, Query, Row,
 use tokio::net::TcpStream;
 use tokio_util::compat::TokioAsyncWriteCompatExt;
 use uuid::Uuid;
+use windmill_common::s3_helpers::convert_json_line_stream;
 use windmill_common::{
     error::{self, to_anyhow, Error},
     utils::empty_as_none,
     worker::{to_raw_value, Connection},
 };
-use windmill_parser_sql::{parse_db_resource, parse_mssql_sig};
+use windmill_parser_sql::{parse_db_resource, parse_mssql_sig, parse_s3_mode};
 use windmill_queue::MiniPulledJob;
 use windmill_queue::{append_logs, CanceledBy};
 
-use crate::common::{build_args_values, OccupancyMetrics};
+use crate::common::{build_args_values, s3_mode_args_to_worker_data, OccupancyMetrics};
 use crate::handle_child::run_future_with_polling_update_job_poller;
 use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
 use crate::AuthedClient;
@@ -63,6 +65,7 @@ pub async fn do_mssql(
     let mssql_args = build_args_values(job, client, conn).await?;
 
     let inline_db_res_path = parse_db_resource(&query);
+    let s3 = parse_s3_mode(&query)?.map(|s3| s3_mode_args_to_worker_data(s3, client.clone(), job));
 
     let db_arg = if let Some(inline_db_res_path) = inline_db_res_path {
         Some(
@@ -197,27 +200,42 @@ pub async fn do_mssql(
         // A response to a query is a stream of data, that must be
         // polled to the end before querying again. Using streams allows
         // fetching data in an asynchronous manner, if needed.
-        let stream = prepared_query.query(&mut client).await.map_err(to_anyhow)?;
 
-        let results = stream.into_results().await.map_err(to_anyhow)?;
-        let len = results.len();
-        let mut json_results = vec![];
-        for (i, statement_result) in results.into_iter().enumerate() {
-            if annotations.return_last_result && i < len - 1 {
-                continue;
-            }
-            let mut json_rows = vec![];
-            for row in statement_result {
-                let row = row_to_json(row)?;
-                json_rows.push(row);
-            }
-            json_results.push(json_rows);
-        }
+        if let Some(s3) = s3 {
+            let rows_stream = async_stream::stream! {
+                let mut stream = prepared_query.query(&mut client).await.map_err(to_anyhow)?.into_row_stream().map(|row| {
+                    row_to_json(row.map_err(to_anyhow)?).map_err(to_anyhow)
+                });
+                while let Some(row) = stream.next().await {
+                    yield row;
+                }
+            };
 
-        if annotations.return_last_result && json_results.len() > 0 {
-            Ok(to_raw_value(&json_results.pop().unwrap()))
+            let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?;
+            s3.upload(stream.boxed()).await?;
+
+            Ok(to_raw_value(&s3.to_return_s3_obj()))
         } else {
-            Ok(to_raw_value(&json_results))
+            let stream = prepared_query.query(&mut client).await.map_err(to_anyhow)?;
+            let results = stream.into_results().await.map_err(to_anyhow)?;
+            let len = results.len();
+            let mut json_results = vec![];
+            for (i, statement_result) in results.into_iter().enumerate() {
+                if annotations.return_last_result && i < len - 1 {
+                    continue;
+                }
+                let mut json_rows = vec![];
+                for row in statement_result {
+                    let row = row_to_json(row)?;
+                    json_rows.push(row);
+                }
+                json_results.push(json_rows);
+            }
+            if annotations.return_last_result && json_results.len() > 0 {
+                Ok(to_raw_value(&json_results.pop().unwrap()))
+            } else {
+                Ok(to_raw_value(&json_results))
+            }
         }
     };
 

backend/windmill-worker/src/mysql_executor.rs

@@ -1,7 +1,8 @@
 use std::{collections::HashMap, sync::Arc};
 
+use anyhow::anyhow;
 use base64::Engine;
-use futures::{future::BoxFuture, FutureExt};
+use futures::{future::BoxFuture, FutureExt, StreamExt};
 use itertools::Itertools;
 use mysql_async::{
     consts::ColumnType, prelude::*, FromValueError, OptsBuilder, Params, Row, SslOpts,
@@ -13,17 +14,18 @@ use std::str::FromStr;
 use tokio::sync::Mutex;
 use windmill_common::{
     error::{to_anyhow, Error},
+    s3_helpers::convert_json_line_stream,
     worker::{to_raw_value, Connection},
 };
 use windmill_parser_sql::{
-    parse_db_resource, parse_mysql_sig, parse_sql_blocks, parse_sql_statement_named_params,
-    RE_ARG_MYSQL_NAMED,
+    parse_db_resource, parse_mysql_sig, parse_s3_mode, parse_sql_blocks,
+    parse_sql_statement_named_params, RE_ARG_MYSQL_NAMED,
 };
 use windmill_queue::CanceledBy;
 use windmill_queue::MiniPulledJob;
 
 use crate::{
-    common::{build_args_values, OccupancyMetrics},
+    common::{build_args_values, s3_mode_args_to_worker_data, OccupancyMetrics, S3ModeWorkerData},
     handle_child::run_future_with_polling_update_job_poller,
     sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args,
     AuthedClient,
@@ -39,12 +41,13 @@ struct MysqlDatabase {
     ssl: Option<bool>,
 }
 
-pub fn do_mysql_inner<'a>(
+fn do_mysql_inner<'a>(
     query: &'a str,
     all_statement_values: &Params,
     conn: Arc<Mutex<mysql_async::Conn>>,
     column_order: Option<&'a mut Option<Vec<String>>>,
     skip_collect: bool,
+    s3: Option<S3ModeWorkerData>,
 ) -> windmill_common::error::Result<BoxFuture<'a, windmill_common::error::Result<Box<RawValue>>>> {
     let param_names = parse_sql_statement_named_params(query, ':')
         .into_iter()
@@ -71,6 +74,38 @@ pub fn do_mysql_inner<'a>(
                 .map_err(to_anyhow)?;
 
             Ok(to_raw_value(&Value::Array(vec![])))
+        } else if let Some(ref s3) = s3 {
+            let query = query.to_string();
+            let rows_stream = async_stream::stream! {
+                let mut conn = conn.lock().await;
+                let mut result = match conn.exec_iter(query, statement_values).await.map_err(to_anyhow) {
+                    Ok(result) => result,
+                    Err(e) => {
+                        yield Err(anyhow!("Error executing query: {:?}", e));
+                        return;
+                    }
+                };
+                loop {
+                    let row = result.next().await;
+                    match row {
+                        Ok(Some(row)) => {
+                            yield Ok(convert_row_to_value(row));
+                        }
+                        Ok(None) => {
+                            break;
+                        }
+                        Err(e) => {
+                            yield Err(anyhow!("Error fetching row: {:?}", e));
+                            return;
+                        }
+                    }
+                }
+            };
+
+            let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?;
+            s3.upload(stream.boxed()).await?;
+
+            Ok(to_raw_value(&s3.to_return_s3_obj()))
         } else {
             let rows: Vec<Row> = conn
                 .lock()
@@ -118,6 +153,7 @@ pub async fn do_mysql(
     let job_args = build_args_values(job, client, conn).await?;
 
     let inline_db_res_path = parse_db_resource(&query);
+    let s3 = parse_s3_mode(&query)?.map(|s3| s3_mode_args_to_worker_data(s3, client.clone(), job));
 
     let db_arg = if let Some(inline_db_res_path) = inline_db_res_path {
         Some(
@@ -252,6 +288,7 @@ pub async fn do_mysql(
                     conn_a.clone(),
                     None,
                     annotations.return_last_result && i < queries.len() - 1,
+                    s3.clone(),
                 )
             })
             .collect::<windmill_common::error::Result<Vec<_>>>()?;
@@ -277,6 +314,7 @@ pub async fn do_mysql(
             conn_a.clone(),
             Some(column_order),
             false,
+            s3,
         )?
     };
 

backend/windmill-worker/src/oracledb_executor.rs

@@ -1,26 +1,31 @@
 use anyhow::anyhow;
 use chrono::Utc;
-use std::{collections::HashMap, str::FromStr, sync::Arc};
+use std::{collections::HashMap, str::FromStr, sync::Arc, vec};
 use windmill_parser::Arg;
 
-use futures::{future::BoxFuture, FutureExt};
+use futures::{future::BoxFuture, FutureExt, StreamExt};
 use itertools::Itertools;
 use oracle::sql_type::{InnerValue, OracleType, ToSql};
 use serde::{Deserialize, Serialize};
 use serde_json::{json, value::RawValue, Value};
 use windmill_common::{
     error::{to_anyhow, Error},
+    s3_helpers::convert_json_line_stream,
     worker::{to_raw_value, Connection},
 };
 use windmill_queue::MiniPulledJob;
 
 use windmill_parser_sql::{
-    parse_db_resource, parse_oracledb_sig, parse_sql_blocks, parse_sql_statement_named_params,
+    parse_db_resource, parse_oracledb_sig, parse_s3_mode, parse_sql_blocks,
+    parse_sql_statement_named_params,
 };
 use windmill_queue::CanceledBy;
 
 use crate::{
-    common::{build_args_values, check_executor_binary_exists, OccupancyMetrics},
+    common::{
+        build_args_values, check_executor_binary_exists, s3_mode_args_to_worker_data,
+        OccupancyMetrics, S3ModeWorkerData,
+    },
     handle_child::run_future_with_polling_update_job_poller,
     sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args,
     AuthedClient,
@@ -43,6 +48,7 @@ pub fn do_oracledb_inner<'a>(
     conn: Arc<std::sync::Mutex<oracle::Connection>>,
     column_order: Option<&'a mut Option<Vec<String>>>,
     skip_collect: bool,
+    s3: Option<S3ModeWorkerData>,
 ) -> windmill_common::error::Result<BoxFuture<'a, windmill_common::error::Result<Box<RawValue>>>> {
     let qw = query.trim_end_matches(';').to_string();
 
@@ -81,55 +87,90 @@ pub fn do_oracledb_inner<'a>(
 
             Ok(to_raw_value(&Value::Array(vec![])))
         } else {
-            let rows = tokio::task::spawn_blocking(move || {
-                let params2: Vec<(&str, &dyn ToSql)> = params
-                    .iter()
-                    .filter(|(k, _)| param_names.contains(&k.clone().into_bytes()))
-                    .map(|(key, val)| (key.as_str(), &**val as &dyn ToSql))
-                    .collect();
+            // We use an mpsc because we need an async stream for s3 mode. However since everything is sync
+            // in rust-oracle, I assumed that calling ResultSet::next() is blocking when it has to refetch.
+            let (tx, rx) = tokio::sync::mpsc::channel::<oracle::Result<Value>>(1000);
+            let (column_order_oneshot_tx, column_order_oneshot_rx) =
+                tokio::sync::oneshot::channel::<Option<Vec<String>>>();
+            let mut column_order_oneshot_tx = Some(column_order_oneshot_tx);
+            let rows_stream = tokio_stream::wrappers::ReceiverStream::new(rx);
+            tokio::task::spawn_blocking(move || {
+                let result = (|| {
+                    let tx = tx.clone();
+                    let params2: Vec<(&str, &dyn ToSql)> = params
+                        .iter()
+                        .filter(|(k, _)| param_names.contains(&k.clone().into_bytes()))
+                        .map(|(key, val)| (key.as_str(), &**val as &dyn ToSql))
+                        .collect();
 
-                let c = conn.lock()?;
-                let mut stmt = c.statement(&qw).build()?;
+                    let c = conn.lock()?;
+                    let mut stmt = c.statement(&qw).build()?;
 
-                let rows = match stmt.statement_type() {
-                    oracle::StatementType::Select => {
-                        let result_rows = stmt.query_named(&params2)?;
-                        let rows: Vec<oracle::Row> =
-                            result_rows.into_iter().filter_map(Result::ok).collect_vec();
-                        rows
-                    }
-                    _ => {
-                        stmt.execute_named(&params2)?;
-                        c.commit()?;
-                        vec![]
-                    }
-                };
+                    match stmt.statement_type() {
+                        oracle::StatementType::Select => {
+                            let mut result_rows = stmt.query_named(&params2)?.enumerate();
+                            while let Some((i, row)) = result_rows.next() {
+                                match row {
+                                    Ok(row) => {
+                                        // If first row, infer column order and send it to the channel
+                                        if i == 0 {
+                                            let col_order: Vec<String> = row
+                                                .column_info()
+                                                .iter()
+                                                .map(|x| x.name().to_string())
+                                                .collect::<Vec<String>>();
+                                            let _ = column_order_oneshot_tx
+                                                .take()
+                                                .unwrap()
+                                                .send(Some(col_order));
+                                        }
 
-                oracle::Result::Ok(rows)
-            })
-            .await
-            .map_err(to_anyhow)?
-            .map_err(to_anyhow)?;
+                                        // called in a spawn_blocking synchronous context, unwrap won't panic
+                                        tx.blocking_send(Ok(convert_row_to_value(row))).unwrap()
+                                    }
+                                    Err(e) => {
+                                        tx.blocking_send(Err(e)).unwrap();
+                                        break;
+                                    }
+                                }
+                            }
+                        }
+                        _ => {
+                            stmt.execute_named(&params2)?;
+                            c.commit()?;
+                        }
+                    };
+                    drop(column_order_oneshot_tx);
+                    Ok::<_, oracle::Error>(())
+                })();
+                match result {
+                    Ok(_) => {}
+                    Err(e) => tx.blocking_send(Err(e)).unwrap(),
+                }
+                // all instances of tx should be dropped here
+            });
 
-            if let Some(column_order) = column_order {
-                *column_order = Some(
-                    rows.first()
-                        .map(|x| {
-                            x.column_info()
-                                .iter()
-                                .map(|x| x.name().to_string())
-                                .collect::<Vec<String>>()
-                        })
-                        .unwrap_or_default(),
-                );
+            if let Ok(Some(col_order)) = column_order_oneshot_rx.await {
+                if let Some(column_order) = column_order {
+                    *column_order = Some(col_order);
+                }
             }
 
-            Ok(to_raw_value(
-                &rows
-                    .into_iter()
-                    .map(|x| convert_row_to_value(x))
-                    .collect::<Vec<serde_json::Value>>(),
-            ))
+            if let Some(s3) = s3 {
+                let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?;
+                s3.upload(stream.boxed()).await?;
+                return Ok(to_raw_value(&s3.to_return_s3_obj()));
+            } else {
+                let rows: Vec<_> = rows_stream.collect().await;
+                Ok(to_raw_value(
+                    &rows
+                        .into_iter()
+                        .collect::<Result<Vec<_>, _>>()
+                        .map_err(to_anyhow)?
+                        .into_iter()
+                        .collect::<Vec<serde_json::Value>>(),
+                ))
+            }
         }
     };
 
@@ -312,6 +353,7 @@ pub async fn do_oracledb(
     let job_args = build_args_values(job, client, conn).await?;
 
     let inline_db_res_path = parse_db_resource(&query);
+    let s3 = parse_s3_mode(&query)?.map(|s3| s3_mode_args_to_worker_data(s3, client.clone(), job));
 
     let db_arg = if let Some(inline_db_res_path) = inline_db_res_path {
         Some(
@@ -376,6 +418,7 @@ pub async fn do_oracledb(
                     conn_a.clone(),
                     None,
                     annotations.return_last_result && i < queries.len() - 1,
+                    s3.clone(),
                 )?
                 .await?;
                 res.push(r);
@@ -390,7 +433,14 @@ pub async fn do_oracledb(
 
         f.boxed()
     } else {
-        do_oracledb_inner(&query, statement_values, conn_a, Some(column_order), false)?
+        do_oracledb_inner(
+            &query,
+            statement_values,
+            conn_a,
+            Some(column_order),
+            false,
+            s3,
+        )?
     };
 
     let result = run_future_with_polling_update_job_poller(

backend/windmill-worker/src/pg_executor.rs

@@ -8,7 +8,7 @@ use anyhow::Context;
 use base64::{engine, Engine as _};
 use chrono::Utc;
 use futures::future::BoxFuture;
-use futures::{FutureExt, TryStreamExt};
+use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt};
 use itertools::Itertools;
 use native_tls::{Certificate, TlsConnector};
 use postgres_native_tls::MakeTlsConnector;
@@ -27,14 +27,18 @@ use tokio_postgres::{
 use uuid::Uuid;
 use windmill_common::error::to_anyhow;
 use windmill_common::error::{self, Error};
+use windmill_common::s3_helpers::convert_json_line_stream;
 use windmill_common::worker::{to_raw_value, Connection, CLOUD_HOSTED};
 use windmill_parser::{Arg, Typ};
 use windmill_parser_sql::{
-    parse_db_resource, parse_pg_statement_arg_indices, parse_pgsql_sig, parse_sql_blocks,
+    parse_db_resource, parse_pg_statement_arg_indices, parse_pgsql_sig, parse_s3_mode,
+    parse_sql_blocks,
 };
 use windmill_queue::{CanceledBy, MiniPulledJob};
 
-use crate::common::{build_args_values, sizeof_val, OccupancyMetrics};
+use crate::common::{
+    build_args_values, s3_mode_args_to_worker_data, sizeof_val, OccupancyMetrics, S3ModeWorkerData,
+};
 use crate::handle_child::run_future_with_polling_update_job_poller;
 use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
 use crate::{AuthedClient, MAX_RESULT_SIZE};
@@ -68,6 +72,7 @@ fn do_postgresql_inner<'a>(
     column_order: Option<&'a mut Option<Vec<String>>>,
     siz: &'a AtomicUsize,
     skip_collect: bool,
+    s3: Option<S3ModeWorkerData>,
 ) -> error::Result<BoxFuture<'a, error::Result<Box<RawValue>>>> {
     let mut query_params = vec![];
 
@@ -106,6 +111,20 @@ fn do_postgresql_inner<'a>(
                 .execute_raw(&query, query_params)
                 .await
                 .map_err(to_anyhow)?;
+        } else if let Some(ref s3) = s3 {
+            let rows_stream = client
+                .query_raw(&query, query_params)
+                .map_err(to_anyhow)
+                .await?
+                .map_err(to_anyhow)
+                .map(|row_result| {
+                    row_result.and_then(|row| postgres_row_to_json_value(row).map_err(to_anyhow))
+                });
+
+            let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?;
+            s3.upload(stream.boxed()).await?;
+
+            return Ok(to_raw_value(&s3.to_return_s3_obj()));
         } else {
             let rows = client
                 .query_raw(&query, query_params)
@@ -172,6 +191,8 @@ pub async fn do_postgresql(
 
     let inline_db_res_path = parse_db_resource(&query);
 
+    let s3 = parse_s3_mode(&query)?.map(|s3| s3_mode_args_to_worker_data(s3, client.clone(), job));
+
     let db_arg = if let Some(inline_db_res_path) = inline_db_res_path {
         Some(
             client
@@ -321,6 +342,7 @@ pub async fn do_postgresql(
                     None,
                     &size,
                     annotations.return_last_result && i < queries.len() - 1,
+                    s3.clone(),
                 )
             })
             .collect::<error::Result<Vec<_>>>()?;
@@ -347,6 +369,7 @@ pub async fn do_postgresql(
             Some(column_order),
             &size,
             false,
+            s3,
         )?
     };
 

backend/windmill-worker/src/result_processor.rs

@@ -29,12 +29,13 @@ use windmill_common::{
 use windmill_common::bench::{BenchmarkInfo, BenchmarkIter};
 
 use windmill_queue::{
-    append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob, WrappedError,
+    append_logs, get_queued_job,  CanceledBy, JobCompleted, MiniPulledJob,
+    WrappedError,
 };
 
 use serde_json::{json, value::RawValue};
 
-use tokio::{sync::broadcast, task::JoinHandle};
+use tokio::task::JoinHandle;
 
 use windmill_queue::{add_completed_job, add_completed_job_error};
 
@@ -43,7 +44,8 @@ use crate::{
     common::{error_to_value, read_result, save_in_cache},
     otel_ee::add_root_flow_job_to_otlp,
     worker_flow::update_flow_status_after_job_completion,
-    AuthedClient, JobCompletedSender, SameWorkerSender, SendResult, INIT_SCRIPT_TAG,
+    AuthedClient, JobCompletedReceiver, JobCompletedSender, SameWorkerSender, SendResult,
+    UpdateFlow, INIT_SCRIPT_TAG,
 };
 
 async fn process_jc(
@@ -118,7 +120,7 @@ async fn process_jc(
 }
 
 pub fn start_background_processor(
-    job_completed_rx: flume::Receiver<SendResult>,
+    job_completed_rx: JobCompletedReceiver,
     job_completed_sender: JobCompletedSender,
     same_worker_queue_size: Arc<AtomicU16>,
     job_completed_processor_is_done: Arc<AtomicBool>,
@@ -127,13 +129,14 @@ pub fn start_background_processor(
     worker_dir: String,
     same_worker_tx: SameWorkerSender,
     worker_name: String,
-    mut killpill_rx: broadcast::Receiver<()>,
     killpill_tx: KillpillSender,
     is_dedicated_worker: bool,
 ) -> JoinHandle<()> {
     tokio::spawn(async move {
         let mut has_been_killed = false;
 
+        let JobCompletedReceiver { bounded_rx, mut killpill_rx, unbounded_rx } = job_completed_rx;
+
         #[cfg(feature = "benchmark")]
         let mut infos = BenchmarkInfo::new();
 
@@ -144,15 +147,21 @@ pub fn start_background_processor(
         //if we have been killed, we want to drain the queue of jobs
         while let Some(sr) = {
             if has_been_killed && same_worker_queue_size.load(Ordering::SeqCst) == 0 {
-                job_completed_rx
+                unbounded_rx
                     .try_recv()
                     .ok()
                     .map(JobCompletedRx::JobCompleted)
+                    .or_else(|| bounded_rx.try_recv().ok().map(JobCompletedRx::JobCompleted))
             } else {
                 tokio::select! {
-                    result = job_completed_rx.recv_async() => {
+                    biased;
+                    result = unbounded_rx.recv_async() => {
                         result.ok().map(JobCompletedRx::JobCompleted)
                     }
+                    result = bounded_rx.recv_async() => {
+                        result.ok().map(JobCompletedRx::JobCompleted)
+                    }
+
                     _ = killpill_rx.recv() => {
                         Some(JobCompletedRx::Killpill)
                     }
@@ -207,7 +216,7 @@ pub fn start_background_processor(
                         infos.add_iter(bench, true);
                     }
                 }
-                JobCompletedRx::JobCompleted(SendResult::UpdateFlow {
+                JobCompletedRx::JobCompleted(SendResult::UpdateFlow(UpdateFlow {
                     flow,
                     w_id,
                     success,
@@ -215,7 +224,7 @@ pub fn start_background_processor(
                     worker_dir,
                     stop_early_override,
                     token,
-                }) => {
+                })) => {
                     // let r;
                     tracing::info!(parent_flow = %flow, "updating flow status");
                     if let Err(e) = update_flow_status_after_job_completion(
@@ -266,29 +275,11 @@ pub fn start_background_processor(
 
 async fn send_job_completed(
     job_completed_tx: JobCompletedSender,
-    job: Arc<MiniPulledJob>,
-    result: Arc<Box<RawValue>>,
-    result_columns: Option<Vec<String>>,
-    mem_peak: i32,
-    canceled_by: Option<CanceledBy>,
-    success: bool,
-    cached_res_path: Option<String>,
-    token: &str,
-    duration: Option<i64>,
+    jc: JobCompleted,
+
 ) {
-    let jc = JobCompleted {
-        job,
-        result,
-        result_columns,
-        mem_peak,
-        canceled_by,
-        success,
-        cached_res_path,
-        token: token.to_string(),
-        duration,
-    };
     job_completed_tx
-        .send_job(jc)
+        .send_job(jc, true)
         .with_context(windmill_common::otel_ee::otel_ctx())
         .await
         .expect("send job completed")
@@ -303,37 +294,28 @@ pub async fn process_result(
     canceled_by: Option<CanceledBy>,
     cached_res_path: Option<String>,
     token: &str,
-    column_order: Option<Vec<String>>,
-    new_args: Option<HashMap<String, Box<RawValue>>>,
+    result_columns: Option<Vec<String>>,
+    preprocessed_args: Option<HashMap<String, Box<RawValue>>>,
     conn: &Connection,
     duration: Option<i64>,
 ) -> error::Result<bool> {
     match result {
-        Ok(r) => {
-            // Update script args to preprocessed args
-            if let Connection::Sql(db) = conn {
-                if let Some(preprocessed_args) = new_args {
-                    sqlx::query!(
-                        "UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2",
-                        Json(preprocessed_args) as Json<HashMap<String, Box<RawValue>>>,
-                        job.id
-                    )
-                    .execute(db)
-                    .await?;
-                }
-            }
+        Ok(result) => {
 
             send_job_completed(
                 job_completed_tx,
-                job,
-                r,
-                column_order,
-                mem_peak,
-                canceled_by,
-                true,
-                cached_res_path,
-                token,
-                duration,
+                JobCompleted {
+                    job,
+                    preprocessed_args,
+                    result,
+                    result_columns,
+                    mem_peak,
+                    canceled_by,
+                    success: true,
+                    cached_res_path,
+                    token: token.to_string(),
+                    duration,
+                },
             )
             .with_context(windmill_common::otel_ee::otel_ctx())
             .await;
@@ -384,15 +366,18 @@ pub async fn process_result(
 
             send_job_completed(
                 job_completed_tx,
-                job,
-                Arc::new(to_raw_value(&error_value)),
-                None,
-                mem_peak,
-                canceled_by,
-                false,
-                cached_res_path,
-                token,
-                duration,
+                JobCompleted {
+                    job,
+                    result: Arc::new(to_raw_value(&error_value)),
+                    result_columns: None,
+                    preprocessed_args: None,
+                    mem_peak,
+                    canceled_by,
+                    success: false,
+                    cached_res_path,
+                    token: token.to_string(),
+                    duration,
+                },
             )
             .with_context(windmill_common::otel_ee::otel_ctx())
             .await;
@@ -468,6 +453,7 @@ pub async fn process_completed_job(
         canceled_by,
         duration,
         result_columns,
+        preprocessed_args,
         ..
     }: JobCompleted,
     client: &AuthedClient,
@@ -492,6 +478,7 @@ pub async fn process_completed_job(
         if job.flow_step_id.as_deref() == Some("preprocessor") {
             // Do this before inserting to `v2_job_completed` for backwards compatibility
             // when we set `flow_status->_metadata->preprocessed_args` to true.
+
             sqlx::query!(
                 r#"UPDATE v2_job SET
                     args = '{"reason":"PREPROCESSOR_ARGS_ARE_DISCARDED"}'::jsonb,
@@ -506,6 +493,15 @@ pub async fn process_completed_job(
                     "error while deleting args of preprocessing step: {e:#}"
                 ))
             })?;
+        } else if let Some(preprocessed_args) = preprocessed_args {
+            // Update script args to preprocessed args
+            sqlx::query!(
+                "UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2",
+                Json(preprocessed_args) as Json<HashMap<String, Box<RawValue>>>,
+                job.id
+            )
+            .execute(db)
+            .await?;
         }
 
         add_time!(bench, "pre add_completed_job");

backend/windmill-worker/src/snowflake_executor.rs

@@ -2,21 +2,28 @@ use base64::{engine, Engine as _};
 use chrono::Datelike;
 use core::fmt::Write;
 use futures::future::BoxFuture;
-use futures::FutureExt;
+use futures::{FutureExt, StreamExt, TryStreamExt};
 use jsonwebtoken::{encode, Algorithm, EncodingKey, Header};
 use reqwest::{Client, Response};
 use serde_json::{json, value::RawValue, Value};
 use sha2::{Digest, Sha256};
 use std::collections::HashMap;
+use windmill_common::error::to_anyhow;
+use windmill_common::s3_helpers::convert_json_line_stream;
 use windmill_common::worker::Connection;
 
 use windmill_common::{error::Error, worker::to_raw_value};
-use windmill_parser_sql::{parse_db_resource, parse_snowflake_sig, parse_sql_blocks};
+use windmill_parser_sql::{
+    parse_db_resource, parse_s3_mode, parse_snowflake_sig, parse_sql_blocks,
+};
 use windmill_queue::{CanceledBy, MiniPulledJob, HTTP_CLIENT};
 
 use serde::{Deserialize, Serialize};
 
-use crate::common::{build_http_client, resolve_job_timeout, OccupancyMetrics};
+use crate::common::{
+    build_http_client, resolve_job_timeout, s3_mode_args_to_worker_data, OccupancyMetrics,
+    S3ModeWorkerData,
+};
 use crate::handle_child::run_future_with_polling_update_job_poller;
 use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
 use crate::{common::build_args_values, AuthedClient};
@@ -123,6 +130,7 @@ fn do_snowflake_inner<'a>(
     column_order: Option<&'a mut Option<Vec<String>>>,
     skip_collect: bool,
     http_client: &'a Client,
+    s3: Option<S3ModeWorkerData>,
 ) -> windmill_common::error::Result<BoxFuture<'a, windmill_common::error::Result<Box<RawValue>>>> {
     let sig = parse_snowflake_sig(&query)
         .map_err(|x| Error::ExecutionErr(x.to_string()))?
@@ -174,7 +182,7 @@ fn do_snowflake_inner<'a>(
                 .parse_snowflake_response::<SnowflakeResponse>()
                 .await?;
 
-            if response.resultSetMetaData.numRows > 10000 {
+            if s3.is_none() && response.resultSetMetaData.numRows > 10000 {
                 return Err(Error::ExecutionErr(
                     "More than 10000 rows were requested, use LIMIT 10000 to limit the number of rows"
                         .to_string(),
@@ -191,54 +199,72 @@ fn do_snowflake_inner<'a>(
                 );
             }
 
-            let mut rows = response.data;
+            // Clones are because, in s3 mode, reqwest::Body::wrap_stream requires the stream to be
+            // 'static even though it doesn't make sense to be in our case since the request is
+            // awaited and the stream is fully read before the function returns.
+            // Turns out it is a real pain to trick the compiler, even using unsafe
+            let cloned_account_identifier: String = account_identifier.to_string();
+            let cloned_token = token.to_string();
 
-            if response.resultSetMetaData.partitionInfo.len() > 1 {
-                for idx in 1..response.resultSetMetaData.partitionInfo.len() {
-                    let url = format!(
-                        "https://{}.snowflakecomputing.com/api/v2/statements/{}",
-                        account_identifier.to_uppercase(),
-                        response.statementHandle
-                    );
-                    let mut request = HTTP_CLIENT
-                        .get(url)
-                        .bearer_auth(token)
-                        .query(&[("partition", idx.to_string())]);
-
-                    if token_is_keypair {
-                        request =
-                            request.header("X-Snowflake-Authorization-Token-Type", "KEYPAIR_JWT");
-                    }
-
-                    let response = request
-                        .send()
-                        .await
-                        .parse_snowflake_response::<SnowflakeDataOnlyResponse>()
-                        .await?;
-
-                    rows.extend(response.data);
+            let rows_stream = async_stream::stream! {
+                for row in response.data {
+                    yield Ok::<Vec<Value>, windmill_common::error::Error>(row);
                 }
+
+                if response.resultSetMetaData.partitionInfo.len() > 1 {
+                    for idx in 1..response.resultSetMetaData.partitionInfo.len() {
+                        let url = format!(
+                            "https://{}.snowflakecomputing.com/api/v2/statements/{}",
+                            cloned_account_identifier.to_uppercase(),
+                            response.statementHandle
+                        );
+                        let mut request = HTTP_CLIENT
+                            .get(url)
+                            .bearer_auth(cloned_token.as_str())
+                            .query(&[("partition", idx.to_string())]);
+
+                        if token_is_keypair {
+                            request =
+                                request.header("X-Snowflake-Authorization-Token-Type", "KEYPAIR_JWT");
+                        }
+
+                        let response = request
+                            .send()
+                            .await
+                            .parse_snowflake_response::<SnowflakeDataOnlyResponse>()
+                            .await?;
+
+                        for row in response.data {
+                            yield Ok(row);
+                        }
+                    }
+                }
+            };
+
+            let rows_stream = rows_stream.map_ok(move |row| {
+                let mut row_map = serde_json::Map::new();
+                row.iter()
+                    .zip(response.resultSetMetaData.rowType.iter())
+                    .for_each(|(val, row_type)| {
+                        row_map.insert(row_type.name.clone(), parse_val(&val, &row_type.r#type));
+                    });
+                row_map
+            });
+
+            if let Some(s3) = s3 {
+                let rows_stream =
+                    rows_stream.map(|r| serde_json::value::to_value(&r?).map_err(to_anyhow));
+                let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?;
+                s3.upload(stream.boxed()).await?;
+                Ok(to_raw_value(&s3.to_return_s3_obj()))
+            } else {
+                let rows = rows_stream
+                    .collect::<Vec<_>>()
+                    .await
+                    .into_iter()
+                    .collect::<Result<Vec<_>, _>>()?;
+                Ok(to_raw_value(&rows))
             }
-
-            let rows = to_raw_value(
-                &rows
-                    .iter()
-                    .map(|row| {
-                        let mut row_map = serde_json::Map::new();
-                        row.iter()
-                            .zip(response.resultSetMetaData.rowType.iter())
-                            .for_each(|(val, row_type)| {
-                                row_map.insert(
-                                    row_type.name.clone(),
-                                    parse_val(&val, &row_type.r#type),
-                                );
-                            });
-                        row_map
-                    })
-                    .collect::<Vec<_>>(),
-            );
-
-            Ok(rows)
         }
     };
 
@@ -259,6 +285,7 @@ pub async fn do_snowflake(
     let snowflake_args = build_args_values(job, client, conn).await?;
 
     let inline_db_res_path = parse_db_resource(&query);
+    let s3 = parse_s3_mode(&query)?.map(|s3| s3_mode_args_to_worker_data(s3, client.clone(), job));
 
     let db_arg = if let Some(inline_db_res_path) = inline_db_res_path {
         Some(
@@ -390,6 +417,7 @@ pub async fn do_snowflake(
                     None,
                     annotations.return_last_result && i < queries.len() - 1,
                     &http_client,
+                    s3.clone(),
                 )
             })
             .collect::<windmill_common::error::Result<Vec<_>>>()?;
@@ -419,6 +447,7 @@ pub async fn do_snowflake(
             Some(column_order),
             false,
             &http_client,
+            s3.clone(),
         )?
     };
     let r = run_future_with_polling_update_job_poller(

backend/windmill-worker/src/worker.rs

@@ -39,7 +39,7 @@ use windmill_common::METRICS_DEBUG_ENABLED;
 #[cfg(feature = "prometheus")]
 use windmill_common::METRICS_ENABLED;
 
-use reqwest::Response;
+use reqwest::{Body, Response};
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use sqlx::types::Json;
 use std::{
@@ -520,6 +520,52 @@ impl AuthedClient {
             _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
         }
     }
+
+    pub async fn upload_s3_file<S>(
+        &self,
+        workspace_id: &str,
+        object_key: String,
+        storage: Option<String>,
+        body: S,
+    ) -> error::Result<()>
+    where
+        S: futures::stream::TryStream + Send + 'static,
+        S::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
+        bytes::Bytes: From<S::Ok>,
+    {
+        let mut query = vec![("file_key", object_key)];
+        if let Some(storage) = storage {
+            query.push(("storage", storage));
+        }
+        let response = self
+            .force_client
+            .as_ref()
+            .unwrap_or(&HTTP_CLIENT)
+            .post(format!(
+                "{}/api/w/{}/job_helpers/upload_s3_file",
+                self.base_internal_url, workspace_id
+            ))
+            .query(&query)
+            .header(
+                reqwest::header::ACCEPT,
+                reqwest::header::HeaderValue::from_static("application/json"),
+            )
+            .header(
+                reqwest::header::AUTHORIZATION,
+                reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))
+                    .map_err(|e| error::Error::BadConfig(e.to_string()))?,
+            )
+            .body(Body::wrap_stream(body))
+            .send()
+            .await
+            .context(format!("Sent upload_s3_file request",))
+            .map_err(error::Error::from)?;
+
+        match response.status().as_u16() {
+            200u16 => Ok(()),
+            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?,
+        }
+    }
 }
 
 #[derive(Clone)]
@@ -528,26 +574,44 @@ pub struct SameWorkerSender(pub Sender<SameWorkerPayload>, pub Arc<AtomicU16>);
 #[allow(dead_code)]
 #[derive(Clone)]
 pub enum JobCompletedSender {
-    Sql(flume::Sender<SendResult>, broadcast::Sender<()>),
+    Sql(SqlJobCompletedSender),
     Http(HttpClient),
     NeverUsed,
 }
 
+#[derive(Clone)]
+pub struct SqlJobCompletedSender {
+    sender: flume::Sender<SendResult>,
+    unbounded_sender: flume::Sender<SendResult>,
+    killpill_tx: broadcast::Sender<()>,
+}
+
+pub struct JobCompletedReceiver {
+    pub bounded_rx: flume::Receiver<SendResult>,
+    pub killpill_rx: broadcast::Receiver<()>,
+    pub unbounded_rx: flume::Receiver<SendResult>,
+}
+
+impl JobCompletedReceiver {
+    pub fn clone(&self) -> Self {
+        Self {
+            bounded_rx: self.bounded_rx.clone(),
+            killpill_rx: self.killpill_rx.resubscribe(),
+            unbounded_rx: self.unbounded_rx.clone(),
+        }
+    }
+}
+
 impl JobCompletedSender {
-    pub fn new(
-        conn: &Connection,
-        buffer_size: usize,
-    ) -> (
-        Self,
-        Option<(flume::Receiver<SendResult>, broadcast::Receiver<()>)>,
-    ) {
+    pub fn new(conn: &Connection, buffer_size: u8) -> (Self, Option<JobCompletedReceiver>) {
         match conn {
             Connection::Sql(_) => {
-                let (sender, receiver) = flume::bounded::<SendResult>(buffer_size);
-                let (killpill_tx, killpill_rx) = broadcast::channel::<()>(buffer_size);
+                let (sender, receiver) = flume::bounded::<SendResult>(buffer_size as usize);
+                let (unbounded_sender, unbounded_rx) = flume::unbounded::<SendResult>();
+                let (killpill_tx, killpill_rx) = broadcast::channel::<()>(10);
                 (
-                    Self::Sql(sender, killpill_tx),
-                    Some((receiver, killpill_rx)),
+                    Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, killpill_tx }),
+                    Some(JobCompletedReceiver { bounded_rx: receiver, killpill_rx, unbounded_rx }),
                 )
             }
             Connection::Http(client) => (Self::Http(client.clone()), None),
@@ -557,14 +621,20 @@ impl JobCompletedSender {
         (Self::NeverUsed, None)
     }
 
-    pub async fn send_job(&self, jc: JobCompleted) -> anyhow::Result<()> {
+    pub async fn send_job(&self, jc: JobCompleted, wait_for_capacity: bool) -> anyhow::Result<()> {
         match self {
-            Self::Sql(sender, _) => sender
+            Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, .. }) => {
+                if wait_for_capacity {
+                    sender
+                } else {
+                    unbounded_sender
+                }
                 .send_async(SendResult::JobCompleted(jc))
                 .await
                 .map_err(|_e| {
                     anyhow::anyhow!("Failed to send job completed to background processor")
-                }),
+                })
+            }
             Self::Http(client) => {
                 crate::agent_workers::send_result(client, jc).await?;
                 Ok(())
@@ -578,9 +648,19 @@ impl JobCompletedSender {
         }
     }
 
-    pub async fn send(&self, send_result: SendResult) -> Result<(), flume::SendError<SendResult>> {
+    pub async fn send(
+        &self,
+        send_result: SendResult,
+        wait_for_capacity: bool,
+    ) -> Result<(), flume::SendError<SendResult>> {
         match self {
-            Self::Sql(sender, _) => sender.send_async(send_result).await,
+            Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, .. }) => {
+                if wait_for_capacity {
+                    sender.send_async(send_result).await
+                } else {
+                    unbounded_sender.send_async(send_result).await
+                }
+            }
             Self::Http(_) => {
                 tracing::error!("Sending job completed to http client, this should not happen");
                 Ok(())
@@ -596,7 +676,7 @@ impl JobCompletedSender {
 
     pub async fn kill(&self) -> Result<(), broadcast::error::SendError<()>> {
         match self {
-            Self::Sql(_, killpill_tx) => {
+            Self::Sql(SqlJobCompletedSender { killpill_tx, .. }) => {
                 tracing::info!("Sending killpill to bg processors");
                 killpill_tx.send(())?;
                 Ok(())
@@ -1017,7 +1097,7 @@ pub async fn run_worker(
 
     let (same_worker_tx, mut same_worker_rx) = mpsc::channel::<SameWorkerPayload>(5);
 
-    let (job_completed_tx, job_completed_rx) = JobCompletedSender::new(&conn, 3);
+    let (job_completed_tx, job_completed_rx) = JobCompletedSender::new(&conn, 10);
 
     let same_worker_queue_size = Arc::new(AtomicU16::new(0));
     let same_worker_tx = SameWorkerSender(same_worker_tx, same_worker_queue_size.clone());
@@ -1025,22 +1105,19 @@ pub async fn run_worker(
         Arc::new(AtomicBool::new(matches!(conn, Connection::Http(_))));
 
     let send_result = match (conn, job_completed_rx) {
-        (Connection::Sql(db), Some((job_completed_rx, bg_killpill_rx))) => {
-            Some(start_background_processor(
-                job_completed_rx,
-                job_completed_tx.clone(),
-                same_worker_queue_size.clone(),
-                job_completed_processor_is_done.clone(),
-                base_internal_url.to_string(),
-                db.clone(),
-                worker_dir.clone(),
-                same_worker_tx.clone(),
-                worker_name.clone(),
-                bg_killpill_rx,
-                killpill_tx.clone(),
-                is_dedicated_worker,
-            ))
-        }
+        (Connection::Sql(db), Some(job_completed_receiver)) => Some(start_background_processor(
+            job_completed_receiver,
+            job_completed_tx.clone(),
+            same_worker_queue_size.clone(),
+            job_completed_processor_is_done.clone(),
+            base_internal_url.to_string(),
+            db.clone(),
+            worker_dir.clone(),
+            same_worker_tx.clone(),
+            worker_name.clone(),
+            killpill_tx.clone(),
+            is_dedicated_worker,
+        )),
         _ => None,
     };
 
@@ -1442,17 +1519,21 @@ pub async fn run_worker(
                 if matches!(job.kind, JobKind::Noop) {
                     add_time!(bench, "send job completed START");
                     job_completed_tx
-                        .send_job(JobCompleted {
-                            job: Arc::new(job.job()),
-                            success: true,
-                            result: Arc::new(empty_result()),
-                            result_columns: None,
-                            mem_peak: 0,
-                            cached_res_path: None,
-                            token: "".to_string(),
-                            canceled_by: None,
-                            duration: None,
-                        })
+                        .send_job(
+                            JobCompleted {
+                                preprocessed_args: None,
+                                job: Arc::new(job.job()),
+                                success: true,
+                                result: Arc::new(empty_result()),
+                                result_columns: None,
+                                mem_peak: 0,
+                                cached_res_path: None,
+                                token: "".to_string(),
+                                canceled_by: None,
+                                duration: None,
+                            },
+                            true,
+                        )
                         .await
                         .expect("send job completed END");
                     add_time!(bench, "sent job completed");
@@ -1666,21 +1747,25 @@ pub async fn run_worker(
                                 }
                                 Connection::Http(_) => {
                                     job_completed_tx
-                                        .send_job(JobCompleted {
-                                            job: arc_job.clone(),
-                                            result: Arc::new(
-                                                windmill_common::worker::to_raw_value(
-                                                    &error_to_value(err),
+                                        .send_job(
+                                            JobCompleted {
+                                                preprocessed_args: None,
+                                                job: arc_job.clone(),
+                                                result: Arc::new(
+                                                    windmill_common::worker::to_raw_value(
+                                                        &error_to_value(err),
+                                                    ),
                                                 ),
-                                            ),
-                                            result_columns: None,
-                                            mem_peak: 0,
-                                            canceled_by: None,
-                                            success: false,
-                                            cached_res_path: None,
-                                            token: authed_client.token.clone(),
-                                            duration: None,
-                                        })
+                                                result_columns: None,
+                                                mem_peak: 0,
+                                                canceled_by: None,
+                                                success: false,
+                                                cached_res_path: None,
+                                                token: authed_client.token.clone(),
+                                                duration: None,
+                                            },
+                                            false,
+                                        )
                                         .await
                                         .expect("send job completed");
                                 }
@@ -1845,15 +1930,17 @@ async fn queue_init_bash_maybe<'c>(
 
 pub enum SendResult {
     JobCompleted(JobCompleted),
-    UpdateFlow {
-        flow: Uuid,
-        w_id: String,
-        success: bool,
-        result: Box<RawValue>,
-        worker_dir: String,
-        stop_early_override: Option<bool>,
-        token: String,
-    },
+    UpdateFlow(UpdateFlow),
+}
+
+pub struct UpdateFlow {
+    pub flow: Uuid,
+    pub w_id: String,
+    pub success: bool,
+    pub result: Box<RawValue>,
+    pub worker_dir: String,
+    pub stop_early_override: Option<bool>,
+    pub token: String,
 }
 
 async fn do_nativets(
@@ -2025,17 +2112,21 @@ async fn handle_queued_job(
                     append_logs(&job.id, &job.workspace_id, logs, conn).await;
                 }
                 job_completed_tx
-                    .send_job(JobCompleted {
-                        job,
-                        result,
-                        result_columns: None,
-                        mem_peak: 0,
-                        canceled_by: None,
-                        success: true,
-                        cached_res_path: None,
-                        token: client.token.clone(),
-                        duration: None,
-                    })
+                    .send_job(
+                        JobCompleted {
+                            preprocessed_args: None,
+                            job,
+                            result,
+                            result_columns: None,
+                            mem_peak: 0,
+                            canceled_by: None,
+                            success: true,
+                            cached_res_path: None,
+                            token: client.token.clone(),
+                            duration: None,
+                        },
+                        true,
+                    )
                     .await
                     .expect("send job completed");
 

backend/windmill-worker/src/worker_flow.rs

@@ -13,8 +13,10 @@ use std::time::Duration;
 
 use crate::common::{cached_result_path, save_in_cache};
 use crate::js_eval::{eval_timeout, IdContext};
+use crate::worker_utils::get_tag_and_concurrency;
 use crate::{
-    AuthedClient, JobCompletedSender, PreviousResult, SameWorkerSender, SendResult, KEEP_JOB_DIR,
+    AuthedClient, JobCompletedSender, PreviousResult, SameWorkerSender, SendResult, UpdateFlow,
+    KEEP_JOB_DIR,
 };
 use anyhow::Context;
 use futures::TryFutureExt;
@@ -58,8 +60,8 @@ use windmill_queue::flow_status::Step;
 use windmill_queue::schedule::get_schedule_opt;
 use windmill_queue::{
     add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job,
-    handle_maybe_scheduled_job, CanceledBy, MiniPulledJob, PushArgs, PushIsolationLevel,
-    SameWorkerPayload, WrappedError,
+    handle_maybe_scheduled_job, insert_concurrency_key, interpolate_args, CanceledBy,
+    MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError,
 };
 
 type DB = sqlx::Pool<sqlx::Postgres>;
@@ -161,6 +163,10 @@ pub async fn update_flow_status_after_job_completion(
                 add_time!(bench, "update flow status internal END");
                 return Ok(None);
             }
+            UpdateFlowStatusAfterJobCompletion::PreprocessingStep => {
+                add_time!(bench, "update flow status preprocessing step END");
+                return Ok(None);
+            }
         }
     }
 }
@@ -170,6 +176,7 @@ pub enum UpdateFlowStatusAfterJobCompletion {
     Done(Arc<MiniPulledJob>),
     NotDone,
     NonLastParallelBranch,
+    PreprocessingStep,
 }
 pub struct RecUpdateFlowStatusAfterJobCompletion {
     flow: uuid::Uuid,
@@ -425,41 +432,6 @@ pub async fn update_flow_status_after_job_completion_internal(
             _ => false,
         };
 
-        if matches!(module_step, Step::PreprocessorStep) {
-            sqlx::query!(
-                "WITH job_result AS (
-                 SELECT result 
-                 FROM v2_job_completed 
-                 WHERE id = $1
-             )
-             UPDATE v2_job 
-             SET args = COALESCE(
-                     CASE 
-                         WHEN job_result.result IS NULL THEN NULL
-                         WHEN jsonb_typeof(job_result.result) = 'object' 
-                         THEN job_result.result
-                         WHEN jsonb_typeof(job_result.result) = 'null'
-                         THEN NULL
-                         ELSE jsonb_build_object('value', job_result.result)
-                     END, 
-                     '{}'::jsonb
-                 ),
-                 preprocessed = TRUE
-             FROM job_result
-             WHERE v2_job.id = $2;
-             ",
-                job_id_for_status,
-                flow
-            )
-            .execute(db)
-            .await
-            .map_err(|e| {
-                Error::internal_err(format!(
-                    "error while updating args in preprocessing step: {e:#}"
-                ))
-            })?;
-        }
-
         let mut tx = db.begin().await?;
 
         add_time!(bench, "process module status START");
@@ -1009,6 +981,128 @@ pub async fn update_flow_status_after_job_completion_internal(
             .ok_or_else(|| Error::internal_err(format!("requiring flow to be in the queue")))?;
         tx.commit().await?;
 
+        if matches!(module_step, Step::PreprocessorStep) {
+            let tag_and_concurrency_key = get_tag_and_concurrency(&flow, db).await;
+            let require_args = tag_and_concurrency_key.as_ref().is_some_and(|x| {
+                x.tag.as_ref().is_some_and(|t| t.contains("$args"))
+                    || x.concurrency_key
+                        .as_ref()
+                        .is_some_and(|ck| ck.contains("$args"))
+            });
+            let mut tag = tag_and_concurrency_key
+                .as_ref()
+                .map(|x| x.tag.clone())
+                .flatten();
+            let concurrency_key = tag_and_concurrency_key
+                .as_ref()
+                .map(|x| x.concurrency_key.clone())
+                .flatten();
+            let concurrent_limit = tag_and_concurrency_key
+                .as_ref()
+                .map(|x| x.concurrent_limit)
+                .flatten();
+            let concurrency_time_window_s = tag_and_concurrency_key
+                .as_ref()
+                .map(|x| x.concurrency_time_window_s)
+                .flatten();
+            if require_args {
+                let args = sqlx::query_scalar!(
+                    "SELECT result  as \"result: Json<HashMap<String, Box<RawValue>>>\"
+                 FROM v2_job_completed 
+                 WHERE id = $1",
+                    job_id_for_status
+                )
+                .fetch_one(db)
+                .await
+                .map_err(|e| {
+                    Error::internal_err(format!("error while fetching preprocessing args: {e:#}"))
+                })?;
+                let args_hm = args.unwrap_or_default().0;
+                let args = PushArgs::from(&args_hm);
+                if let Some(ck) = concurrency_key {
+                    let mut tx = db.begin().await?;
+                    insert_concurrency_key(
+                        &flow_job.workspace_id,
+                        &args,
+                        &flow_job.runnable_path,
+                        JobKind::Flow,
+                        Some(ck),
+                        &mut tx,
+                        flow,
+                    )
+                    .await?;
+                    tx.commit().await?;
+                }
+                if let Some(t) = tag {
+                    tag = Some(interpolate_args(t, &args, &flow_job.workspace_id));
+                }
+            } else if let Some(ck) = concurrency_key {
+                let mut tx = db.begin().await?;
+                insert_concurrency_key(
+                    &flow_job.workspace_id,
+                    &PushArgs::from(&HashMap::new()),
+                    &flow_job.runnable_path,
+                    JobKind::Flow,
+                    Some(ck),
+                    &mut tx,
+                    flow,
+                )
+                .await?;
+                tx.commit().await?;
+            }
+
+            // let tag = tag_and_concurrency_key.and_then(|tc| tc.tag.map(|t| interpolate_args(t.clone(), &args, &workspace_id)));
+            // let concurrency_key = tag_and_concurrency_key.and_then(|tc| tc.concurrency_key.map(|ck| interpolate_args(&ck, &args, &workspace_id)));
+            sqlx::query!(
+                "WITH job_result AS (
+                 SELECT result 
+                 FROM v2_job_completed 
+                 WHERE id = $1
+             ),
+             updated_queue AS (
+                UPDATE v2_job_queue
+                SET running = false,
+                tag = COALESCE($3, tag)
+                WHERE id = $2
+             )
+             UPDATE v2_job 
+             SET 
+                tag = COALESCE($3, tag),
+                concurrent_limit = COALESCE($4, concurrent_limit),
+                concurrency_time_window_s = COALESCE($5, concurrency_time_window_s),
+                args = COALESCE(
+                     CASE 
+                         WHEN job_result.result IS NULL THEN NULL
+                         WHEN jsonb_typeof(job_result.result) = 'object' 
+                         THEN job_result.result
+                         WHEN jsonb_typeof(job_result.result) = 'null'
+                         THEN NULL
+                         ELSE jsonb_build_object('value', job_result.result)
+                     END, 
+                     '{}'::jsonb
+                 ),
+                 preprocessed = TRUE
+             FROM job_result
+             WHERE v2_job.id = $2;
+             ",
+                job_id_for_status,
+                flow,
+                tag,
+                concurrent_limit,
+                concurrency_time_window_s,
+            )
+            .execute(db)
+            .await
+            .map_err(|e| {
+                Error::internal_err(format!(
+                    "error while updating args in preprocessing step: {e:#}"
+                ))
+            })?;
+            if success {
+                return Ok(UpdateFlowStatusAfterJobCompletion::PreprocessingStep);
+            }
+        }
+
         let job_root = flow_job
             .flow_innermost_root_job
             .map(|x| x.to_string())
@@ -1539,22 +1633,47 @@ pub async fn handle_flow(
             );
         }
     }
-    let mut rec = Some(PushNextFlowJobRec { flow_job: flow_job, status: status });
-    while let Some(nrec) = rec {
-        rec = push_next_flow_job(
-            nrec.flow_job,
-            nrec.status,
+    let mut rec = PushNextFlowJobRec { flow_job: flow_job, status: status };
+    loop {
+        let PushNextFlowJobRec { flow_job, status } = rec;
+        let next = push_next_flow_job(
+            flow_job,
+            status,
             flow,
             db,
             client,
             last_result.clone(),
             same_worker_tx.clone(),
             worker_dir,
-            job_completed_tx.clone(),
             worker_name,
         )
         .warn_after_seconds(10)
         .await?;
+        match next {
+            PushNextFlowJob::Rec(nrec) => {
+                tracing::info!("recursively pushing next flow job {}", nrec.flow_job.id);
+                rec = nrec;
+            }
+            PushNextFlowJob::Done(update_flow) => {
+                if let Some(update_flow) = update_flow {
+                    tracing::info!(
+                        "sending flow status update {} with success {} to job completed channel",
+                        update_flow.flow,
+                        update_flow.success
+                    );
+                    job_completed_tx
+                        .send(SendResult::UpdateFlow(update_flow), false)
+                        .warn_after_seconds(3)
+                        .await
+                        .map_err(|e| {
+                            Error::internal_err(format!(
+                                "error sending update flow message to job completed channel: {e:#}"
+                            ))
+                        })?;
+                }
+                break;
+            }
+        }
     }
 
     Ok(())
@@ -1600,6 +1719,10 @@ lazy_static::lazy_static! {
     pub static ref EHM: HashMap<String, Box<RawValue>> = HashMap::new();
 }
 
+enum PushNextFlowJob {
+    Rec(PushNextFlowJobRec),
+    Done(Option<UpdateFlow>),
+}
 struct PushNextFlowJobRec {
     flow_job: Arc<MiniPulledJob>,
     status: FlowStatus,
@@ -1615,9 +1738,8 @@ async fn push_next_flow_job(
     last_job_result: Option<Arc<Box<RawValue>>>,
     same_worker_tx: SameWorkerSender,
     worker_dir: &str,
-    job_completed_tx: JobCompletedSender,
     worker_name: &str,
-) -> error::Result<Option<PushNextFlowJobRec>> {
+) -> error::Result<PushNextFlowJob> {
     let job_root = flow_job
         .flow_innermost_root_job
         .map(|x| x.to_string())
@@ -1650,29 +1772,20 @@ async fn push_next_flow_job(
 
     // if this is an empty module of if the module has already been completed, successfully, update the parent flow
     if flow.modules.is_empty() || matches!(status_module, FlowStatusModule::Success { .. }) {
-        job_completed_tx
-            .send(SendResult::UpdateFlow {
-                flow: flow_job.id,
-                success: true,
-                result: if flow.modules.is_empty() {
-                    to_raw_value(arc_flow_job_args.as_ref())
-                } else {
-                    // it has to be an empty for loop event
-                    serde_json::from_str("[]").unwrap()
-                },
-                stop_early_override: None,
-                w_id: flow_job.workspace_id.clone(),
-                worker_dir: worker_dir.to_string(),
-                token: client.token.clone(),
-            })
-            .await
-            .map_err(|e| {
-                Error::internal_err(format!(
-                    "error sending update flow message to job completed channel: {e:#}"
-                ))
-            })?;
-
-        return Ok(None);
+        return Ok(PushNextFlowJob::Done(Some(UpdateFlow {
+            flow: flow_job.id,
+            success: true,
+            result: if flow.modules.is_empty() {
+                to_raw_value(arc_flow_job_args.as_ref())
+            } else {
+                // it has to be an empty for loop event
+                serde_json::from_str("[]").unwrap()
+            },
+            stop_early_override: None,
+            w_id: flow_job.workspace_id.clone(),
+            worker_dir: worker_dir.to_string(),
+            token: client.token.clone(),
+        })));
     }
 
     if matches!(step, Step::Step(0)) {
@@ -1684,6 +1797,7 @@ async fn push_next_flow_job(
                 flow_job.workspace_id.as_str()
             )
             .fetch_one(db)
+            .warn_after_seconds(3)
             .await?;
             if no_flow_overlap {
                 let overlapping = sqlx::query_scalar!(
@@ -1704,6 +1818,7 @@ async fn push_next_flow_job(
                      flow_job.runnable_path()
                  )
                  .fetch_all(db)
+                 .warn_after_seconds(3)
                  .await?;
                 if overlapping.len() > 0 {
                     let overlapping_str = overlapping
@@ -1711,27 +1826,21 @@ async fn push_next_flow_job(
                         .map(|x| x.to_string())
                         .collect::<Vec<String>>()
                         .join(", ");
-                    job_completed_tx
-                         .send(SendResult::UpdateFlow {
-                             flow: flow_job.id,
-                             success: true,
-                             result: serde_json::from_str(
-                                 &format!("\"not allowed to overlap with {overlapping_str}, scheduling next iteration\""),
-                             )
-                             .unwrap(),
-                             stop_early_override: Some(true),
-                             w_id: flow_job.workspace_id.clone(),
-                             worker_dir: worker_dir.to_string(),
-                             token: client.token.clone(),
-                         })
-                         .await
-                         .map_err(|e| {
-                             Error::internal_err(format!(
-                                 "error sending update flow message to job completed channel: {e:#}"
-                             ))
-                         })?;
 
-                    return Ok(None);
+                    return Ok(PushNextFlowJob::Done(Some(
+                        UpdateFlow {
+                            flow: flow_job.id,
+                            success: true,
+                            result: serde_json::from_str(
+                                &format!("\"not allowed to overlap with {overlapping_str}, scheduling next iteration\""),
+                            )
+                            .unwrap(),
+                            stop_early_override: Some(true),
+                            w_id: flow_job.workspace_id.clone(),
+                            worker_dir: worker_dir.to_string(),
+                            token: client.token.clone(),
+                        }
+                    )));
                 }
             }
         }
@@ -1749,26 +1858,18 @@ async fn push_next_flow_job(
                     flow_job.scheduled_for.to_string(),
                 )]),
             )
+            .warn_after_seconds(3)
             .await?;
             if skip {
-                job_completed_tx
-                    .send(SendResult::UpdateFlow {
-                        flow: flow_job.id,
-                        success: true,
-                        result: serde_json::from_str("\"stopped early\"").unwrap(),
-                        stop_early_override: Some(true),
-                        w_id: flow_job.workspace_id.clone(),
-                        worker_dir: worker_dir.to_string(),
-                        token: client.token.clone(),
-                    })
-                    .await
-                    .map_err(|e| {
-                        Error::internal_err(format!(
-                            "error sending update flow message to job completed channel: {e:#}"
-                        ))
-                    })?;
-
-                return Ok(None);
+                return Ok(PushNextFlowJob::Done(Some(UpdateFlow {
+                    flow: flow_job.id,
+                    success: true,
+                    result: serde_json::from_str("\"stopped early\"").unwrap(),
+                    stop_early_override: Some(true),
+                    w_id: flow_job.workspace_id.clone(),
+                    worker_dir: worker_dir.to_string(),
+                    token: client.token.clone(),
+                })));
             }
         }
     }
@@ -1787,7 +1888,10 @@ async fn push_next_flow_job(
         if last_job_result.is_some() {
             last_job_result.unwrap()
         } else {
-            match get_previous_job_result(db, flow_job.workspace_id.as_str(), &status).await? {
+            match get_previous_job_result(db, flow_job.workspace_id.as_str(), &status)
+                .warn_after_seconds(3)
+                .await?
+            {
                 None => Arc::new(to_raw_value(&json!("{}"))),
                 Some(previous_job_result) => Arc::new(previous_job_result),
             }
@@ -1806,7 +1910,7 @@ async fn push_next_flow_job(
         FlowStatusModule::WaitingForPriorSteps { .. } | FlowStatusModule::WaitingForEvents { .. }
     ) {
         if let Some((suspend, last)) = needs_resume(&flow, &status) {
-            let mut tx = db.begin().await?;
+            let mut tx = db.begin().warn_after_seconds(3).await?;
 
             /* Lock this row to prevent the suspend column getting out out of sync
              * if a resume message arrives after we fetch and count them here.
@@ -1817,6 +1921,7 @@ async fn push_next_flow_job(
                 flow_job.id
             )
             .fetch_one(&mut *tx)
+            .warn_after_seconds(3)
             .await
             .context("lock flow in queue")?;
 
@@ -1825,7 +1930,9 @@ async fn push_next_flow_job(
              )
              .bind(last)
              .fetch_all(&mut *tx)
-             .await?
+             .warn_after_seconds(3)
+             .await
+             ?
              .into_iter()
              .collect::<Vec<_>>();
 
@@ -1865,6 +1972,7 @@ async fn push_next_flow_job(
                                      None,
                                      None
                                  )
+                                 .warn_after_seconds(3)
                                  .await
                                  .map_err(|e| {
                                      Error::ExecutionErr(format!(
@@ -1897,6 +2005,7 @@ async fn push_next_flow_job(
                     flow_job.id
                 )
                 .execute(&mut *tx)
+                .warn_after_seconds(3)
                 .await?;
             }
 
@@ -1938,6 +2047,7 @@ async fn push_next_flow_job(
                          Some(&serde_json::json!({"approved": false, "job_id": flow_job.id, "details": "Suspend timed out without approval but can continue".to_string()}).to_string()),
                          None,
                      )
+                     .warn_after_seconds(3)
                      .await?;
                 }
 
@@ -1959,6 +2069,7 @@ async fn push_next_flow_job(
                      flow_job.id
                  )
                  .execute(&mut *tx)
+                 .warn_after_seconds(3)
                  .await?;
 
                 // Remove the approval conditions from the flow status
@@ -1969,10 +2080,11 @@ async fn push_next_flow_job(
                     flow_job.id
                 )
                 .execute(&mut *tx)
+                .warn_after_seconds(3)
                 .await?;
 
                 /* continue on and run this job! */
-                tx.commit().await?;
+                tx.commit().warn_after_seconds(3).await?;
 
             /* not enough messages to do this job, "park"/suspend until there are */
             } else if matches!(
@@ -2002,6 +2114,7 @@ async fn push_next_flow_job(
                     flow_job.id,
                 )
                 .execute(&mut *tx)
+                .warn_after_seconds(3)
                 .await?;
 
                 sqlx::query!(
@@ -2010,10 +2123,11 @@ async fn push_next_flow_job(
                     flow_job.id,
                 )
                 .execute(&mut *tx)
+                .warn_after_seconds(3)
                 .await?;
 
-                tx.commit().await?;
-                return Ok(None);
+                tx.commit().warn_after_seconds(3).await?;
+                return Ok(PushNextFlowJob::Done(None));
 
             /* cancelled or we're WaitingForEvents but we don't have enough messages (timed out) */
             } else {
@@ -2027,9 +2141,10 @@ async fn push_next_flow_job(
                          Some(&serde_json::json!({"approved": false, "job_id": flow_job.id, "details": "Suspend timed out without approval and is cancelled".to_string()}).to_string()),
                          None,
                      )
+                     .warn_after_seconds(3)
                      .await?;
                 }
-                tx.commit().await?;
+                tx.commit().warn_after_seconds(3).await?;
 
                 let (logs, error_name) = if let Some(disapprover) = is_disapproved {
                     (
@@ -2057,26 +2172,18 @@ async fn push_next_flow_job(
                     logs.clone(),
                     &db.into(),
                 )
+                .warn_after_seconds(3)
                 .await;
 
-                job_completed_tx
-                    .send(SendResult::UpdateFlow {
-                        flow: flow_job.id,
-                        success: false,
-                        result: to_raw_value(&result),
-                        stop_early_override: None,
-                        w_id: flow_job.workspace_id.clone(),
-                        worker_dir: worker_dir.to_string(),
-                        token: client.token.clone(),
-                    })
-                    .await
-                    .map_err(|e| {
-                        Error::internal_err(format!(
-                            "error sending update flow message to job completed channel: {e:#}"
-                        ))
-                    })?;
-
-                return Ok(None);
+                return Ok(PushNextFlowJob::Done(Some(UpdateFlow {
+                    flow: flow_job.id,
+                    success: false,
+                    result: to_raw_value(&result),
+                    stop_early_override: None,
+                    w_id: flow_job.workspace_id.clone(),
+                    worker_dir: worker_dir.to_string(),
+                    token: client.token.clone(),
+                })));
             }
         }
     }
@@ -2137,6 +2244,7 @@ async fn push_next_flow_job(
                                  None,
                                  None,
                              )
+                             .warn_after_seconds(3)
                              .await
                              .map_err(|e| {
                                  Error::ExecutionErr(format!(
@@ -2237,6 +2345,7 @@ async fn push_next_flow_job(
                         flow_job.id
                     )
                     .execute(db)
+                    .warn_after_seconds(3)
                     .await
                     .context("update flow retry")?;
                 };
@@ -2255,7 +2364,9 @@ async fn push_next_flow_job(
     drop(resume_messages);
 
     let is_skipped = if let Some(skip_if) = &module.skip_if {
-        let idcontext = get_transform_context(&flow_job, previous_id.as_str(), &status).await?;
+        let idcontext = get_transform_context(&flow_job, previous_id.as_str(), &status)
+            .warn_after_seconds(3)
+            .await?;
         compute_bool_from_expr(
             &skip_if.expr,
             arc_flow_job_args.clone(),
@@ -2266,6 +2377,7 @@ async fn push_next_flow_job(
             Some((resumes.clone(), resume.clone(), approvers.clone())),
             None,
         )
+        .warn_after_seconds(3)
         .await?
     } else {
         false
@@ -2295,6 +2407,7 @@ async fn push_next_flow_job(
                 &flow_job.workspace_id
             )
             .fetch_optional(db)
+            .warn_after_seconds(3)
             .await?;
             if let Some(args) = args {
                 Ok(Marc::new(args.map(|x| x.0).unwrap_or_else(HashMap::new)))
@@ -2327,7 +2440,9 @@ async fn push_next_flow_job(
                     | FlowModuleValue::FlowScript { input_transforms, .. }
                     | FlowModuleValue::Flow { input_transforms, .. },
                 ) => {
-                    let ctx = get_transform_context(&flow_job, &previous_id, &status).await?;
+                    let ctx = get_transform_context(&flow_job, &previous_id, &status)
+                        .warn_after_seconds(3)
+                        .await?;
                     transform_context = Some(ctx);
                     let by_id = transform_context.as_ref().unwrap();
                     transform_input(
@@ -2340,6 +2455,7 @@ async fn push_next_flow_job(
                         by_id,
                         client,
                     )
+                    .warn_after_seconds(3)
                     .await
                     .map(Marc::new)
                 }
@@ -2370,6 +2486,7 @@ async fn push_next_flow_job(
         approvers.clone(),
         is_skipped,
     )
+    .warn_after_seconds(3)
     .await?;
     tracing::info!(id = %flow_job.id, root_id = %job_root, "next flow transform computed");
 
@@ -2395,6 +2512,7 @@ async fn push_next_flow_job(
                 flow_job.id
             )
             .fetch_optional(db)
+            .warn_after_seconds(3)
             .await?
             .flatten();
 
@@ -2404,13 +2522,13 @@ async fn push_next_flow_job(
 
             if let Some(status) = status {
                 // // flow is reprocessed by the worker in a state where the module has completed successfully.
-                return Ok(Some(PushNextFlowJobRec {
+                return Ok(PushNextFlowJob::Rec(PushNextFlowJobRec {
                     flow_job: flow_job,
                     status: status,
                 }));
             } else {
                 return Err(Error::BadRequest(
-                    "impossible to parse new flow status after applying innr flows".to_string(),
+                    "impossible to parse new flow status after applying inner flows".to_string(),
                 ));
             }
         }
@@ -2433,7 +2551,7 @@ async fn push_next_flow_job(
     };
     let len = job_payloads.len();
 
-    let mut tx = db.begin().await?;
+    let mut tx = db.begin().warn_after_seconds(3).await?;
     let nargs = args.as_ref();
     for (i, payload_tag) in job_payloads.into_iter().enumerate() {
         if i % 100 == 0 && i != 0 {
@@ -2443,6 +2561,7 @@ async fn push_next_flow_job(
                 flow_job.id,
             )
             .execute(db)
+            .warn_after_seconds(3)
             .await?;
         }
         tracing::debug!(id = %flow_job.id, root_id = %job_root, "pushing job {i} of {len}");
@@ -2481,7 +2600,9 @@ async fn push_next_flow_job(
                 args.insert("iter".to_string(), to_raw_value(new_args));
                 if let Some(input_transforms) = simple_input_transforms {
                     //previous id is none because we do not want to use previous id if we are in a for loop
-                    let ctx = get_transform_context(&flow_job, "", &status).await?;
+                    let ctx = get_transform_context(&flow_job, "", &status)
+                        .warn_after_seconds(3)
+                        .await?;
                     let ti = transform_input(
                         Marc::new(args),
                         arc_last_job_result.clone(),
@@ -2492,6 +2613,7 @@ async fn push_next_flow_job(
                         &ctx,
                         client,
                     )
+                    .warn_after_seconds(3)
                     .await
                     .map_err(|e| {
                         Error::ExecutionErr(
@@ -2527,7 +2649,9 @@ async fn push_next_flow_job(
                         to_raw_value(&json!({ "index": i as i32, "value": itered[i]})),
                     );
                     if let Some(input_transforms) = simple_input_transforms {
-                        let ctx = get_transform_context(&flow_job, &previous_id, &status).await?;
+                        let ctx = get_transform_context(&flow_job, &previous_id, &status)
+                            .warn_after_seconds(3)
+                            .await?;
                         let ti = transform_input(
                             Marc::new(hm),
                             arc_last_job_result.clone(),
@@ -2538,6 +2662,7 @@ async fn push_next_flow_job(
                             &ctx,
                             client,
                         )
+                        .warn_after_seconds(3)
                         .await
                         .map_err(|e| {
                             Error::ExecutionErr(format!(
@@ -2608,6 +2733,7 @@ async fn push_next_flow_job(
                      flow_job.workspace_id,
                  )
                  .fetch_optional(&mut *tx)
+                 .warn_after_seconds(3)
                  .await?
                  .map(|x| x.into())
             } else {
@@ -2616,8 +2742,8 @@ async fn push_next_flow_job(
         };
 
         tracing::debug!(id = %flow_job.id, root_id = %job_root, "computed perms for job {i} of {len}");
-        let tag = if flow_job.tag == "flow"
-            || flow_job.tag == format!("flow-{}", flow_job.workspace_id)
+        let tag = if !matches!(step, Step::PreprocessorStep)
+            && (flow_job.tag == "flow" || flow_job.tag == format!("flow-{}", flow_job.workspace_id))
         {
             payload_tag.tag.clone()
         } else {
@@ -2668,6 +2794,7 @@ async fn push_next_flow_job(
                 worker_name
             )
             .execute(&mut *inner_tx)
+            .warn_after_seconds(3)
             .await;
         }
 
@@ -2688,6 +2815,7 @@ async fn push_next_flow_job(
                         uuid,
                     )
                     .execute(&mut *inner_tx)
+                    .warn_after_seconds(3)
                     .await?;
                 }
                 tracing::debug!(id = %flow_job.id, root_id = %job_root, "updated suspend for {uuid}");
@@ -2707,6 +2835,7 @@ async fn push_next_flow_job(
                  root_job.unwrap_or(flow_job.id)
              )
              .execute(&mut *inner_tx)
+             .warn_after_seconds(3)
              .await?;
         }
 
@@ -2731,6 +2860,7 @@ async fn push_next_flow_job(
                 uuid
             )
             .execute(&mut *tx)
+            .warn_after_seconds(3)
             .await?;
             tracing::debug!(id = %flow_job.id, root_id = %job_root, "updated parallel monitor lock for {uuid}");
         }
@@ -2844,6 +2974,7 @@ async fn push_next_flow_job(
                 flow_job.id
             )
             .execute(&mut *tx)
+            .warn_after_seconds(3)
             .await?;
         }
         Step::PreprocessorStep => {
@@ -2860,6 +2991,7 @@ async fn push_next_flow_job(
                 flow_job.id
             )
             .execute(&mut *tx)
+            .warn_after_seconds(3)
             .await?;
         }
         Step::Step(i) => {
@@ -2877,6 +3009,7 @@ async fn push_next_flow_job(
                 flow_job.id
             )
             .execute(&mut *tx)
+            .warn_after_seconds(3)
             .await?;
         }
     };
@@ -2888,6 +3021,7 @@ async fn push_next_flow_job(
         flow_job.id
     )
     .execute(&mut *tx)
+    .warn_after_seconds(3)
     .await?;
 
     if continue_on_same_worker {
@@ -2903,10 +3037,11 @@ async fn push_next_flow_job(
     if continue_on_same_worker {
         same_worker_tx
             .send(SameWorkerPayload { job_id: first_uuid, recoverable: true })
+            .warn_after_seconds(3)
             .await
             .map_err(to_anyhow)?;
     }
-    return Ok(None);
+    return Ok(PushNextFlowJob::Done(None));
 }
 
 // async fn jump_to_next_step(

backend/windmill-worker/src/worker_utils.rs

@@ -3,13 +3,14 @@ use tracing::Instrument;
 use uuid::Uuid;
 use windmill_common::{
     agent_workers::{PingJobStatus, PingJobStatusResponse},
+    cache,
     worker::{
         get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage,
         insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query,
         update_worker_ping_main_loop_query, Connection, Ping, PingType, WORKER_CONFIG,
         WORKER_GROUP,
     },
-    KillpillSender,
+    KillpillSender, DB,
 };
 
 use crate::{
@@ -320,3 +321,71 @@ pub(crate) async fn queue_vacuum(conn: &Connection, worker_name: &str, hostname:
         }
     }
 }
+
+#[derive(Debug, Clone, sqlx::FromRow)]
+pub struct TagAndConcurrencyKey {
+    pub tag: Option<String>,
+    pub concurrency_key: Option<String>,
+    pub concurrent_limit: Option<i32>,
+    pub concurrency_time_window_s: Option<i32>,
+    pub version: Option<i64>,
+}
+
+pub async fn get_tag_and_concurrency(job_id: &Uuid, db: &DB) -> Option<TagAndConcurrencyKey> {
+    let r = sqlx::query_as!(
+        TagAndConcurrencyKey,
+        "
+        WITH j AS (
+            SELECT 
+                raw_flow->>'concurrency_key' as concurrency_key, 
+                raw_flow->>'concurrency_time_window_s' as concurrency_time_window_s,
+                raw_flow->>'concurrency_limit' as concurrent_limit,
+                runnable_path, 
+                runnable_id as version FROM v2_job
+            WHERE id = $1
+        )
+        SELECT tag, j.concurrency_key, j.concurrency_time_window_s::int, j.concurrent_limit::int, j.version
+            FROM flow, j
+            WHERE path = j.runnable_path
+        ",
+        job_id
+    )
+    .fetch_optional(db)
+    .await
+    .ok()
+    .flatten();
+    if let Some(tag_and_concurrency_key) = r {
+        if tag_and_concurrency_key.concurrency_key.as_ref().is_some()
+            || tag_and_concurrency_key.version.as_ref().is_none()
+        {
+            return Some(tag_and_concurrency_key);
+        } else {
+            let version = tag_and_concurrency_key.version.unwrap();
+
+            let r = cache::flow::fetch_version_lite(db, version).await;
+            let flow = match r {
+                Ok(data) => Ok(data),
+                Err(_) => cache::flow::fetch_version(db, version).await,
+            };
+            let flow_value = flow.map(|f| f.value().clone()).ok();
+            let concurrency_key = flow_value
+                .as_ref()
+                .map(|fv| fv.concurrency_key.clone())
+                .flatten();
+            let concurrent_limit = flow_value.as_ref().map(|fv| fv.concurrent_limit).flatten();
+            let concurrent_time_window_s = flow_value
+                .as_ref()
+                .map(|fv| fv.concurrency_time_window_s)
+                .flatten();
+            Some(TagAndConcurrencyKey {
+                tag: tag_and_concurrency_key.tag,
+                concurrency_key,
+                concurrent_limit,
+                concurrency_time_window_s: concurrent_time_window_s,
+                version: None,
+            })
+        }
+    } else {
+        None
+    }
+}

benchmarks/lib.ts

@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
 import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
 import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
 
-export const VERSION = "v1.490.0";
+export const VERSION = "v1.491.5";
 
 export async function login(email: string, password: string): Promise<string> {
   return await windmill.UserService.login({

cli/main.ts

@@ -63,7 +63,7 @@ export {
 //   }
 // });
 
-export const VERSION = "1.490.0";
+export const VERSION = "1.491.5";
 
 const command = new Command()
   .name("wmill")
@@ -94,6 +94,7 @@ const command = new Command()
     "Specify headers to use for all requests. e.g: \"HEADERS='h1: v1, h2: v2'\""
   )
   .version(VERSION)
+  .versionOption(false)
   .command("init", "Bootstrap a windmill project with a wmill.yaml file")
   .action(async () => {
     if (await Deno.stat("wmill.yaml").catch(() => null)) {
@@ -134,15 +135,34 @@ const command = new Command()
   .command("worker-groups", workerGroups)
   .command("workers", workers)
   .command("queues", queues)
-  .command("version", "Show version information")
+  .command("version --version", "Show version information")
   .action(async (opts) => {
-    console.log("CLI build against " + VERSION);
+    console.log("CLI version: " + VERSION);
+    try {
+      const provider = new NpmProvider({ package: "windmill-cli" });
+      const versions = await provider.getVersions("windmill-cli");
+      if (versions.latest !== VERSION) {
+        console.log(
+          `CLI is outdated. Latest version ${versions.latest} is available. Run \`wmill upgrade\` to update.`
+        );
+      } else {
+        console.log("CLI is up to date");
+      }
+    } catch (e) {
+      console.warn(
+        `Cannot fetch latest CLI version on npmjs to check if up-to-date: ${e}`
+      );
+    }
     const workspace = await getActiveWorkspace(opts as GlobalOptions);
     if (workspace) {
-      const backendVersion = await fetchVersion(workspace.remote);
-      console.log("Backend Version: " + backendVersion);
+      try {
+        const backendVersion = await fetchVersion(workspace.remote);
+        console.log("Backend Version: " + backendVersion);
+      } catch (e) {
+        console.warn("Cannot fetch backend version: " + e);
+      }
     } else {
-      console.log(
+      console.warn(
         "Cannot fetch backend version: no active workspace selected, choose one to pick a remote to fetch version of"
       );
     }

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
 	"name": "windmill-components",
-	"version": "1.490.0",
+	"version": "1.491.5",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "windmill-components",
-			"version": "1.490.0",
+			"version": "1.491.5",
 			"hasInstallScript": true,
 			"license": "AGPL-3.0",
 			"dependencies": {

frontend/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "windmill-components",
-	"version": "1.490.0",
+	"version": "1.491.5",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",

frontend/src/lib/components/LogViewer.svelte

@@ -69,7 +69,7 @@
 			? truncateContent.substring(
 					s3LogPrefixes[prefixIndex]?.length,
 					end == -1 ? undefined : end + 1
-			  )
+				)
 			: undefined
 	}
 
@@ -109,7 +109,7 @@
 			? truncatedContent.substring(
 					truncatedContent.substring(1).indexOf('\n') + 2,
 					truncatedContent.length
-			  )
+				)
 			: truncatedContent
 	)
 	export function scrollToBottom() {

frontend/src/lib/components/QueueMetricsDrawer.svelte

@@ -2,7 +2,7 @@
 	import { onMount } from 'svelte'
 	import { Drawer, DrawerContent, Button } from './common'
 	import QueueMetricsDrawerInner from './QueueMetricsDrawerInner.svelte'
-	import { ConfigService } from '$lib/gen'
+	import { ConfigService, type Alert } from '$lib/gen'
 	import Section from './Section.svelte'
 	import { sendUserToast } from '$lib/toast'
 	import { Pencil, Trash, Check, PlusCircle, SaveIcon } from 'lucide-svelte'
@@ -21,14 +21,6 @@
 		}
 	}
 
-	type Alert = {
-		name: string
-		tags_to_monitor: string[]
-		jobs_num_threshold: number
-		alert_cooldown_seconds: number
-		alert_time_threshold_seconds: number
-	}
-
 	let drawer: Drawer
 	export function openDrawer() {
 		drawer?.openDrawer()
@@ -55,8 +47,8 @@
 
 	async function fetchConfig() {
 		try {
-			const response = (await ConfigService.getConfig({ name: configName })) as { alerts: Alert[] }
-			alerts = response.alerts || []
+			const response = await ConfigService.getConfig({ name: configName })
+			alerts = response?.alerts || []
 			originalAlerts = JSON.parse(JSON.stringify(alerts))
 		} catch (error) {
 			console.error('Failed to fetch config:', error)
@@ -298,7 +290,9 @@
 														<input
 															type="text"
 															bind:value={newTag}
-															placeholder="{workerTags.length === alert.tags_to_monitor.length ? 'All tags already added' : 'Add tag from dropdown' }"
+															placeholder={workerTags.length === alert.tags_to_monitor.length
+																? 'All tags already added'
+																: 'Add tag from dropdown'}
 															on:input={(e) => filterTags(e)}
 															disabled={workerTags.length === alert.tags_to_monitor.length}
 															class="p-1 flex-grow mr-1"
@@ -321,15 +315,15 @@
 														>
 															{#each filteredTags as tag}
 																{#if !alert.tags_to_monitor.includes(tag)}
-																<li>
-																	<button
-																		type="button"
-																		class="w-full text-left p-2 cursor-pointer hover:bg-slate-200 dark:hover:bg-slate-700"
-																		on:click={() => addTag(index, tag)}
-																	>
-																		{tag}
-																	</button>
-																</li>
+																	<li>
+																		<button
+																			type="button"
+																			class="w-full text-left p-2 cursor-pointer hover:bg-slate-200 dark:hover:bg-slate-700"
+																			on:click={() => addTag(index, tag)}
+																		>
+																			{tag}
+																		</button>
+																	</li>
 																{/if}
 															{/each}
 														</ul>

frontend/src/lib/components/S3FilePicker.svelte

@@ -41,9 +41,9 @@
 	export let fromWorkspaceSettings: boolean = false
 	export let readOnlyMode: boolean
 
-	export let initialFileKey: { s3: string } | undefined = undefined
-	let initialFileKeyInternalCopy: { s3: string }
-	export let selectedFileKey: { s3: string } | undefined = undefined
+	export let initialFileKey: { s3: string; storage?: string } | undefined = undefined
+	let initialFileKeyInternalCopy: { s3: string; storage?: string }
+	export let selectedFileKey: { s3: string; storage?: string } | undefined = undefined
 	export let folderOnly = false
 	export let regexFilter: RegExp | undefined = undefined
 
@@ -298,7 +298,7 @@
 			deletionModalOpen = false
 		}
 		sendUserToast(`${fileKey} deleted from S3 bucket`)
-		selectedFileKey = { s3: '' }
+		selectedFileKey = { s3: '', storage }
 		const currentPage = page
 		await clearAndLoadFiles()
 		for (let i = 0; i < currentPage; i++) {
@@ -359,7 +359,7 @@
 			moveModalOpen = false
 		}
 		sendUserToast(`${srcFileKey} moved to ${destFileKey}`)
-		selectedFileKey = { s3: destFileKey! }
+		selectedFileKey = { s3: destFileKey!, storage }
 		await clearAndLoadFiles()
 		await loadFileMetadataPlusPreviewAsync(selectedFileKey.s3)
 	}
@@ -397,7 +397,7 @@
 		await clearAndLoadFiles()
 		if (selectedFileKey !== undefined) {
 			if (allFilesByKey[selectedFileKey.s3] === undefined) {
-				selectedFileKey = { s3: '' }
+				selectedFileKey = { s3: '', storage }
 			} else {
 				loadFileMetadataPlusPreviewAsync(selectedFileKey.s3)
 			}
@@ -421,7 +421,8 @@
 		if (item.type === 'folder') {
 			if (folderOnly) {
 				selectedFileKey = {
-					s3: item_key
+					s3: item_key,
+					storage
 				}
 			}
 			if (toggleCollapsed) {
@@ -456,7 +457,8 @@
 			displayedFileKeys = displayedFileKeys.sort()
 		} else {
 			selectedFileKey = {
-				s3: item_key
+				s3: item_key,
+				storage
 			}
 			loadFileMetadataPlusPreviewAsync(selectedFileKey.s3)
 		}
@@ -841,7 +843,7 @@
 	on:close={async (evt) => {
 		uploadModalOpen = false
 		if (evt.detail !== undefined && evt.detail !== null) {
-			selectedFileKey = { s3: evt.detail }
+			selectedFileKey = { s3: evt.detail, storage }
 			await clearAndLoadFiles()
 			loadFileMetadataPlusPreviewAsync(evt.detail)
 		}

frontend/src/lib/components/apps/components/display/table/AppAggridTable.svelte

@@ -303,8 +303,8 @@
 
 	function getIdFromData(data: any): string {
 		return resolvedConfig?.rowIdCol && resolvedConfig?.rowIdCol != ''
-			? (data?.[resolvedConfig?.rowIdCol] ?? data['__index'])
-			: data['__index']
+			? (data?.[resolvedConfig?.rowIdCol] ?? data?.['__index'])
+			: data?.['__index']
 	}
 
 	function mountGrid() {
@@ -389,8 +389,24 @@
 										e.api.deselectAll()
 										outputs?.selectedRow?.set({})
 										outputs?.selectedRowIndex.set(0)
-									} else {
-										e.api.getRowNode(index.toString())?.setSelected(true)
+									} else if (Array.isArray(index)) {
+										// select all rows matching the indixes
+										e.api.deselectAll()
+										index.forEach((i) => {
+											let rowId = getIdFromData(value[i])
+											if (rowId) {
+												e.api.getRowNode(rowId)?.setSelected(true, false)
+											}
+										})
+									} else if (typeof index === 'number') {
+										let rowId = getIdFromData(value[index])
+										if (rowId) {
+											e.api.getRowNode(rowId)?.setSelected(true, true)
+											outputs?.selectedRowIndex.set(index)
+											const row = { ...value[index] }
+											delete row['__index']
+											outputs?.selectedRow?.set(row)
+										}
 									}
 								},
 								setValue(nvalue) {

frontend/src/lib/components/apps/components/layout/AppTabs.svelte

@@ -199,24 +199,22 @@
 							<span class="mr-2 w-8 font-mono">{selected == res ? '-' : '+'}</span>
 							{res}
 						</button>
-						{#if selected == res}
-							<div class="border-t">
-								<SubGridEditor
-									{id}
-									visible={render && index === selectedIndex}
-									subGridId={`${id}-${index}`}
-									class={twMerge(css?.container?.class, 'wm-tabs-container')}
-									style={css?.container?.style}
-									containerHeight={componentContainerHeight - (titleBarHeight * tabs.length + 40)}
-									on:focus={() => {
-										if (!$connectingInput.opened) {
-											$selectedComponent = [id]
-											handleTabSelection()
-										}
-									}}
-								/>
-							</div>
-						{/if}
+						<div class={selected == res ? 'border-t' : ''}>
+							<SubGridEditor
+								{id}
+								visible={render && index === selectedIndex}
+								subGridId={`${id}-${index}`}
+								class={twMerge(css?.container?.class, 'wm-tabs-container')}
+								style={css?.container?.style}
+								containerHeight={componentContainerHeight - (titleBarHeight * tabs.length + 40)}
+								on:focus={() => {
+									if (!$connectingInput.opened) {
+										$selectedComponent = [id]
+										handleTabSelection()
+									}
+								}}
+							/>
+						</div>
 					</div>
 				{/each}
 			</div>

frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte

@@ -55,7 +55,7 @@
 		bun: 'TypeScript',
 		php: 'PHP',
 		rust: 'Rust',
-		ansible: 'Ansible Playbook',
+		ansible: 'Ansible',
 		csharp: 'C#',
 		nu: 'Nu',
 		java: 'Java'

frontend/src/lib/components/search/GlobalSearchModal.svelte

@@ -262,6 +262,7 @@
 	}
 
 	let selectedItem: any
+
 	async function handleKeydown(event: KeyboardEvent) {
 		if ((!isMac() ? event.ctrlKey : event.metaKey) && event.key === 'k') {
 			event.preventDefault()
@@ -641,11 +642,11 @@
 								<div class="w-4/12 overflow-y-auto max-h-[70vh]">
 									{#each itemMap['runs'] ?? [] as r}
 										<QuickMenuItem
-											on:hover={() => {
+											on:select={() => {
 												selectedItem = r
 												selectedWorkspace = r?.document.workspace_id[0]
 											}}
-											on:select={() => {
+											on:keyboardOnlySelect={() => {
 												open = false
 												goto(`/run/${r?.document.id[0]}`)
 											}}
@@ -657,10 +658,7 @@
 										>
 											<svelte:fragment slot="itemReplacement">
 												<div
-													class={twMerge(
-														`w-full flex flex-row items-center gap-4 transition-all`,
-														r?.document.id === selectedItem?.document?.id ? 'bg-surface-hover' : ''
-													)}
+													class="w-full flex flex-row items-center gap-4 transition-all"
 												>
 													<div
 														class="rounded-full w-2 h-2 {r?.document.success[0]

frontend/src/lib/components/search/QuickMenuItem.svelte

@@ -24,6 +24,7 @@
 	async function handleKeydown(event: KeyboardEvent) {
 		if (hovered && event.key === 'Enter') {
 			event.preventDefault()
+			dispatch('keyboardOnlySelect')
 			runAction()
 		}
 	}

frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte

@@ -263,7 +263,7 @@
 								btnClasses="ml-4 mt-2"
 								color="dark"
 								size="xs"
-								href={itemKind === 'flow' ? '/flows/add?hub=68' : '/scripts/add?hub=hub%2F11446'}
+								href={itemKind === 'flow' ? '/flows/add?hub=68' : '/scripts/add?hub=hub%2F19662'}
 								target="_blank">Create from template</Button
 							>
 						{/if}

frontend/src/lib/components/triggers/http/RouteEditorInner.svelte

@@ -268,6 +268,7 @@
 		bind:selectedFileKey={static_asset_config}
 		on:close={() => {
 			s3Editor?.setCode(JSON.stringify(static_asset_config, null, 2))
+			s3FileUploadRawMode = true
 		}}
 		readOnlyMode={false}
 	/>
@@ -366,6 +367,7 @@
 										disabled={!can_write}
 									/>
 								{/if}
+								{s3FileUploadRawMode}
 								{#if s3FileUploadRawMode}
 									{#if can_write}
 										<JsonEditor
@@ -396,7 +398,6 @@
 													s3: evt.detail?.path ?? '',
 													filename: evt.detail?.filename ?? undefined
 												}
-												s3FileUploadRawMode = true
 											}}
 											on:deletion={(evt) => {
 												static_asset_config = {
@@ -447,7 +448,7 @@
 										size="xs"
 										href={itemKind === 'flow'
 											? '/flows/add?hub=62'
-											: '/scripts/add?hub=hub%2F11627'}
+											: '/scripts/add?hub=hub%2F19669'}
 										target="_blank">Create from template</Button
 									>
 								{/if}
@@ -622,7 +623,9 @@
 								{/if}
 							{/if}
 
-							<RouteBodyTransformerOption bind:raw_string bind:wrap_body />
+							{#if !static_asset_config}
+								<RouteBodyTransformerOption bind:raw_string bind:wrap_body />
+							{/if}
 						</div>
 					</Section>
 				{/if}

frontend/src/lib/components/triggers/http/utils.ts

@@ -2,7 +2,7 @@ import { base } from '$lib/base'
 import { isCloudHosted } from '$lib/cloud'
 
 export const SECRET_KEY_PATH = 'secret_key_path'
-export const HUB_SCRIPT_ID = 19661
+export const HUB_SCRIPT_ID = 19670
 export const SIGNATURE_TEMPLATE_SCRIPT_HUB_PATH: string = `hub/${HUB_SCRIPT_ID}`
 export const SIGNATURE_TEMPLATE_FLOW_HUB_ID = '67'
 

frontend/src/lib/script_helpers.ts

@@ -249,6 +249,7 @@ export async function main(message: string, name: string, step_id: string) {
 `
 
 const POSTGRES_INIT_CODE = `-- to pin the database use '-- database f/your/path'
+-- to stream a large query result to your workspace storage use '-- s3'
 -- to only return the result of the last query use '--return_last_result'
 -- $1 name1 = default arg
 -- $2 name2
@@ -259,6 +260,7 @@ UPDATE demo SET col2 = \$4::INT WHERE col2 = \$2::INT;
 `
 
 const MYSQL_INIT_CODE = `-- to pin the database use '-- database f/your/path'
+-- to stream a large query result to your workspace storage use '-- s3'
 -- :name1 (text) = default arg
 -- :name2 (int)
 -- :name3 (int)
@@ -267,6 +269,7 @@ UPDATE demo SET col2 = :name3 WHERE col2 = :name2;
 `
 
 const BIGQUERY_INIT_CODE = `-- to pin the database use '-- database f/your/path'
+-- to stream a large query result to your workspace storage use '-- s3'
 -- @name1 (string) = default arg
 -- @name2 (integer)
 -- @name3 (string[])
@@ -276,6 +279,7 @@ UPDATE \`demodb.demo\` SET col2 = @name4 WHERE col2 = @name2;
 `
 
 const ORACLEDB_INIT_CODE = `-- to pin the database use '-- database f/your/path'
+-- to stream a large query result to your workspace storage use '-- s3'
 -- :name1 (text) = default arg
 -- :name2 (int)
 -- :name3 (int)
@@ -284,6 +288,7 @@ UPDATE demo SET col2 = :name3 WHERE col2 = :name2;
 `
 
 const SNOWFLAKE_INIT_CODE = `-- to pin the database use '-- database f/your/path'
+-- to stream a large query result to your workspace storage use '-- s3'
 -- ? name1 (varchar) = default arg
 -- ? name2 (int)
 INSERT INTO demo VALUES (?, ?);
@@ -294,6 +299,7 @@ UPDATE demo SET col2 = ? WHERE col2 = ?;
 
 const MSSQL_INIT_CODE = `-- return_last_result
 -- to pin the database use '-- database f/your/path'
+-- to stream a large query result to your workspace storage use '-- s3'
 -- @P1 name1 (varchar) = default arg
 -- @P2 name2 (int)
 -- @P3 name3 (int)
@@ -654,86 +660,94 @@ export const TS_PREPROCESSOR_FLOW_INTRO = `/**
 export const TS_PREPROCESSOR_MODULE_CODE = `export async function preprocessor(
   event:
     | {
-        kind: "webhook";
-        body: any,
-        raw_string: string | null,
-        query: Record<string, string>;
-        headers: Record<string, string>;
-      }
+      kind: "webhook";
+      body: any;
+      raw_string: string | null;
+      query: Record<string, string>;
+      headers: Record<string, string>;
+    }
     | {
-        kind: "http";
-        body: any,
-        raw_string: string | null,
-        route: string;
-        path: string;
-        method: string;
-        params: Record<string, string>;
-        query: Record<string, string>;
-        headers: Record<string, string>;
-      }
+      kind: "http";
+      body: any;
+      raw_string: string | null;
+      route: string;
+      path: string;
+      method: string;
+      params: Record<string, string>;
+      query: Record<string, string>;
+      headers: Record<string, string>;
+    }
     | {
-        kind: "email";
-        parsed_email: any,
-        raw_email: string,
-      }
+      kind: "email";
+      parsed_email: any;
+      raw_email: string;
+    }
     | { kind: "websocket"; msg: string; url: string }
     | {
-        kind: "kafka";
-        payload: string;
-        brokers: string[];
-        topic: string;
-        group_id: string;
-      }
+      kind: "kafka";
+      payload: string;
+      brokers: string[];
+      topic: string;
+      group_id: string;
+    }
     | {
-        kind: "nats";
-        payload: string;
-        servers: string[];
-        subject: string;
-        headers?: Record<string, string[]>;
-        status?: number;
-        description?: string;
-        length: number;
-      }
+      kind: "nats";
+      payload: string;
+      servers: string[];
+      subject: string;
+      headers?: Record<string, string[]>;
+      status?: number;
+      description?: string;
+      length: number;
+    }
     | {
-        kind: "sqs";
-        msg: string,
-        queue_url: string;
-        message_id?: string;
-        receipt_handle?: string;
-        attributes: Record<string, string>;
-        message_attributes?: Record<
-          string,
-          { string_value?: string; data_type: string }
-        >;
-      }
+      kind: "sqs";
+      msg: string;
+      queue_url: string;
+      message_id?: string;
+      receipt_handle?: string;
+      attributes: Record<string, string>;
+      message_attributes?: Record<
+        string,
+        { string_value?: string; data_type: string }
+      >;
+    }
     | {
-        kind: "mqtt";
-        payload: string,
-        topic: string;
-        retain: boolean;
-        pkid: number;
-        qos: number;
-        v5?: {
-          payload_format_indicator?: number;
-          topic_alias?: number;
-          response_topic?: string;
-          correlation_data?: Array<number>;
-          user_properties?: Array<[string, string]>;
-          subscription_identifiers?: Array<number>;
-          content_type?: string;
-        };
-      }
+      kind: "mqtt";
+      payload: string;
+      topic: string;
+      retain: boolean;
+      pkid: number;
+      qos: number;
+      v5?: {
+        payload_format_indicator?: number;
+        topic_alias?: number;
+        response_topic?: string;
+        correlation_data?: Array<number>;
+        user_properties?: Array<[string, string]>;
+        subscription_identifiers?: Array<number>;
+        content_type?: string;
+      };
+    }
     | {
-        kind: "gcp";
-        payload: string,
-        message_id: string;
-        subscription: string;
-        ordering_key?: string;
-        attributes?: Record<string, string>;
-        delivery_type: "push" | "pull";
-        headers?: Record<string, string>;
-        publish_time?: string;
-      }
+      kind: "gcp";
+      payload: string;
+      message_id: string;
+      subscription: string;
+      ordering_key?: string;
+      attributes?: Record<string, string>;
+      delivery_type: "push" | "pull";
+      headers?: Record<string, string>;
+      publish_time?: string;
+    }
+    | {
+      kind: "postgres";
+      transaction_type: "insert" | "update" | "delete",
+      schema_name: string,
+      table_name: string,
+      old_row?: Record<string, any>,
+      row: Record<string, any>
+    }
 ) {
   return {
     // return the args to be passed to the runnable
@@ -892,6 +906,16 @@ class GcpEvent(TypedDict):
     headers: Optional[dict[str, str]]
     publish_time: Optional[str]
 
+
+class PostgresEvent(TypedDict):
+    kind: Literal["postgres"]
+    transaction_type: Literal["insert", "update", "delete"]
+    schema_name: str
+    table_name: str
+    old_row: Optional[dict[str, any]]
+    row: dict[str, any]
+
+
 Event = Union[
     WebhookEvent,
     HttpEvent,
@@ -902,6 +926,7 @@ Event = Union[
     SqsEvent,
     MqttEvent,
     GcpEvent,
+    PostgresEvent,
 ]
 
 

frontend/src/lib/scripts.ts

@@ -136,7 +136,7 @@ const scriptLanguagesArray: [SupportedLanguage | 'docker' | 'bunnative', string]
 	['powershell', 'PowerShell'],
 	['php', 'PHP'],
 	['rust', 'Rust'],
-	['ansible', 'Ansible Playbook'],
+	['ansible', 'Ansible'],
 	['csharp', 'C#'],
 	['docker', 'Docker'],
 	['nu', 'Nu'],