diff --git a/.aiderignore b/.aiderignore new file mode 100644 index 0000000000..0dca6eac77 --- /dev/null +++ b/.aiderignore @@ -0,0 +1,3 @@ +/* +!/backend/ +!/frontend/ \ No newline at end of file diff --git a/.github/workflows/aider-after-review.yaml b/.github/workflows/aider-after-review.yaml new file mode 100644 index 0000000000..6ae15bfc18 --- /dev/null +++ b/.github/workflows/aider-after-review.yaml @@ -0,0 +1,170 @@ +name: Aider Auto-fix PR Review Change Requests + +on: + pull_request_review: + types: [submitted] + +jobs: + auto-fix-review: + if: github.event.review.state == 'changes_requested' && contains(github.event.pull_request.title, '[Aider PR]') + runs-on: ubicloud-standard-8 + permissions: + contents: write + pull-requests: write + env: + GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WINDMILL_TOKEN: ${{ secrets.WINDMILL_TOKEN }} + + steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + + - name: Check out code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Configure Git User + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Checkout PR Branch + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "PR review trigger: Checking out PR branch..." + PR_NUMBER=${{ github.event.pull_request.number }} + PR_HEAD_REF=$(gh pr view $PR_NUMBER --json headRefName -q .headRefName --repo $GITHUB_REPOSITORY) + if [[ -z "$PR_HEAD_REF" || "$PR_HEAD_REF" == "null" ]]; then + echo "::error::Could not determine PR head branch for PR #$PR_NUMBER via gh CLI." + exit 1 + fi + echo "Checking out PR head branch: $PR_HEAD_REF for PR #$PR_NUMBER" + git fetch origin "refs/heads/${PR_HEAD_REF}:refs/remotes/origin/${PR_HEAD_REF}" --no-tags + git checkout "$PR_HEAD_REF" + echo "Successfully checked out branch $(git rev-parse --abbrev-ref HEAD)" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Aider and Dependencies + run: | + python -m pip install aider-install; aider-install + pip install -U google-generativeai + sudo apt-get update && sudo apt-get install -y jq + + - name: Generate Prompt from Review + id: generate_prompt + shell: bash + run: | + mkdir -p .github/aider + PROMPT_FILE_PATH=".github/aider/review-prompt.txt" + + # Get PR review body + REVIEW_BODY="${{ github.event.review.body }}" + PR_NUMBER="${{ github.event.pull_request.number }}" + + # Get PR description for context NOT USED FOR NOW + # PR_DETAILS=$(gh pr view $PR_NUMBER --json title,body --repo $GITHUB_REPOSITORY) + # PR_TITLE=$(echo "$PR_DETAILS" | jq -r .title) + # PR_BODY=$(echo "$PR_DETAILS" | jq -r .body) + + # Get all PR review comments + REVIEW_COMMENTS=$(gh pr view $PR_NUMBER --json reviews -q '.reviews[] | select(.state == "CHANGES_REQUESTED") | .body' --repo $GITHUB_REPOSITORY) + REVIEW_BODY_Q=$(printf '%q' "$REVIEW_BODY") + + # Update query to get review comments from all review types, not just "CHANGES_REQUESTED" + ALL_REVIEW_COMMENTS=$(gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/comments \ + | jq '[.[] | {diff_hunk: .diff_hunk, path: .path, body: .body}]') + + BASE_PROMPT="Fix the following issues in the PR based on the review feedback. The review body is prepended with REVIEW. The review comments are prepended with REVIEW_COMMENTS. The review body and comments are separated by a blank line." + printf "%s\nREVIEW:\n%s\nREVIEW_COMMENTS:\n%s" \ + "$BASE_PROMPT" "$REVIEW_BODY_Q" "$ALL_REVIEW_COMMENTS" > "$PROMPT_FILE_PATH" + echo "PROMPT_FILE_PATH=$PROMPT_FILE_PATH" >> $GITHUB_OUTPUT + + - name: Run Aider with review prompt + run: | + aider \ + --read .cursor/rules/rust-best-practices.mdc \ + --read .cursor/rules/svelte5-best-practices.mdc \ + --model gemini/gemini-2.5-pro-preview-05-06 \ + --message-file .github/aider/review-prompt.txt \ + --yes \ + --no-check-update \ + --auto-commits \ + --no-analytics \ + --no-gitignore \ + | tee .github/aider/aider-output.txt || true + echo "Aider command completed. Output saved to .github/aider/aider-output.txt" + # Check if there are any changes to commit + if [[ -z "$(git status --porcelain)" ]]; then + echo "No changes detected after running Aider." + exit 0 + fi + + - name: Clean up prompt file + if: always() + run: rm -f .github/aider/review-prompt.txt + + - name: Commit and Push Changes + id: commit_and_push + if: ${{ success() }} + run: | + CURRENT_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) + echo "Attempting to push changes to PR branch $CURRENT_BRANCH_NAME for PR #${{ github.event.pull_request.number }}" + + # Pull latest changes to avoid rejection due to non-fast-forward + git pull origin $CURRENT_BRANCH_NAME + + if git push origin $CURRENT_BRANCH_NAME; then + echo "Push to $CURRENT_BRANCH_NAME successful." + echo "CHANGES_APPLIED=true" >> $GITHUB_OUTPUT + else + echo "::warning::Push to PR branch $CURRENT_BRANCH_NAME failed." + echo "CHANGES_APPLIED=false" >> $GITHUB_OUTPUT + fi + + - name: Comment on PR + if: success() + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUM: ${{ github.event.pull_request.number }} + run: | + # Create comment body in a temporary file to avoid command line length limits + if [[ "${{ steps.commit_and_push.outputs.CHANGES_APPLIED }}" == "true" ]]; then + cat > /tmp/pr-comment.md << EOL + 🤖 I've automatically addressed the feedback based on the review. + + ## Aider Output + \`\`\` + $(cat .github/aider/aider-output.txt || echo 'No output available') + \`\`\` + + Please review the changes and let me know if further adjustments are needed. + EOL + else + cat > /tmp/pr-comment.md << EOL + 🤖 I attempted to address the review feedback, but no modifications were made. + + ## Aider Output + \`\`\` + $(cat .github/aider/aider-output.txt || echo 'No output available') + \`\`\` + + Please review the output and provide additional guidance if needed. + EOL + fi + + # Use the file for comment body + gh pr comment $PR_NUM --body-file /tmp/pr-comment.md diff --git a/.github/workflows/aider.yaml b/.github/workflows/aider.yaml new file mode 100644 index 0000000000..bc337b640e --- /dev/null +++ b/.github/workflows/aider.yaml @@ -0,0 +1,342 @@ +name: Aider Auto-fix issues and PR comments via external prompt + +on: + issue_comment: + types: [created] + +jobs: + auto-fix: + runs-on: ubicloud-standard-8 + if: | + github.event_name == 'issue_comment' && + contains(github.event.comment.body, '/aider') && + !contains(github.event.comment.user.login, '[bot]') + permissions: + contents: write + pull-requests: write + issues: write + env: + GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WINDMILL_TOKEN: ${{ secrets.WINDMILL_TOKEN }} + + steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + + - name: Check out code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Configure Git User + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Checkout PR Branch + if: github.event_name == 'issue_comment' && github.event.issue.pull_request + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "Issue comment trigger: Checking out PR branch..." + PR_NUMBER=${{ github.event.issue.number }} + PR_HEAD_REF=$(gh pr view $PR_NUMBER --json headRefName -q .headRefName --repo $GITHUB_REPOSITORY) + if [[ -z "$PR_HEAD_REF" || "$PR_HEAD_REF" == "null" ]]; then + echo "::error::Could not determine PR head branch for PR #$PR_NUMBER via gh CLI." + exit 1 + fi + echo "Checking out PR head branch: $PR_HEAD_REF for PR #$PR_NUMBER" + git fetch origin "refs/heads/${PR_HEAD_REF}:refs/remotes/origin/${PR_HEAD_REF}" --no-tags + git checkout "$PR_HEAD_REF" + echo "Successfully checked out branch $(git rev-parse --abbrev-ref HEAD)" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Aider and Dependencies + run: | + python -m pip install aider-install; aider-install + pip install -U google-generativeai + sudo apt-get update && sudo apt-get install -y jq + + - name: Determine Prompt for Aider + id: determine_prompt + shell: bash + run: | + PROMPT_FILE_PATH=".github/aider/issue-prompt.txt" + mkdir -p .github/aider + + # Determine if this is a PR comment or regular issue comment + if [[ ! -z "${{ github.event.issue.pull_request }}" ]]; then + echo "This is a comment on a Pull Request" + PR_NUMBER="${{ github.event.issue.number }}" + + # Get PR description to check for issue references + PR_BODY=$(gh pr view $PR_NUMBER --json body -q .body --repo $GITHUB_REPOSITORY) + + # Extract issue number from PR description (looking for #123 or "fixes #123" patterns) + REFERENCED_ISSUE=$(echo "$PR_BODY" | grep -oE "#[0-9]+" | grep -oE "[0-9]+" | head -1) + + if [[ ! -z "$REFERENCED_ISSUE" ]]; then + echo "Found referenced issue #$REFERENCED_ISSUE in PR description" + + # Fetch the referenced issue details + ISSUE_DETAILS=$(gh issue view $REFERENCED_ISSUE --json title,body --repo $GITHUB_REPOSITORY) + ISSUE_TITLE=$(echo "$ISSUE_DETAILS" | jq -r .title) + ISSUE_BODY=$(echo "$ISSUE_DETAILS" | jq -r .body) + + # Store raw comment body in a file first to avoid shell interpretation issues + echo '${{ github.event.comment.body }}' > /tmp/raw_comment.txt + RAW_COMMENT_BODY=$(cat /tmp/raw_comment.txt) + # Remove the /aider prefix and trim whitespace + COMMENT_CONTENT=$(echo "$RAW_COMMENT_BODY" | sed 's|^/aider||' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + + echo "Sending issue content and PR comment to external API…" + + ISSUE_TITLE_Q=$(printf '%q' "$ISSUE_TITLE") + ISSUE_BODY_Q=$(printf '%q' "$ISSUE_BODY") + + JSON_PAYLOAD=$(jq -n \ + --arg title "$ISSUE_TITLE_Q" \ + --arg body "$ISSUE_BODY_Q" \ + '{"body":{"issue_title":$title,"issue_body":$body}}') + + API_RESULT=$(curl -s -w "\n%{http_code}" \ + -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run_wait_result/p/f/ai/quiet_script" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $WINDMILL_TOKEN" \ + --data-binary "$JSON_PAYLOAD" \ + --max-time 90) + + HTTP_CODE=$(echo "$API_RESULT" | tail -n1) + BODY=$(echo "$API_RESULT" | sed '$d') + + echo "$BODY" > /tmp/api_response.txt + + BASE_PROMPT="Try to fix the following issue based on the instruction given by the user. The issue is prepended with the word ISSUE. The instruction is prepended with the word INSTRUCTION. The issue and instruction are separated by a blank line." + if [[ "$HTTP_CODE" -eq 200 ]]; then + PROCESSED_ISSUE_PROMPT=$(jq -r '.effective_body // empty' /tmp/api_response.txt) + if [[ -z "$PROCESSED_ISSUE_PROMPT" || "$PROCESSED_ISSUE_PROMPT" == "null" ]]; then + PROCESSED_ISSUE_PROMPT="" + fi + printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \ + "$BASE_PROMPT" "$PROCESSED_ISSUE_PROMPT" "$COMMENT_CONTENT" > "$PROMPT_FILE_PATH" + else + echo "::warning::API call failed (HTTP $HTTP_CODE). Using PR comment with issue context." + printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \ + "$BASE_PROMPT" "$ISSUE_BODY_Q" "$COMMENT_CONTENT" > "$PROMPT_FILE_PATH" + fi + rm -f /tmp/api_response.txt + else + echo "No referenced issue found in PR description, using comment content only" + # Use comment content directly as with regular issue comments + echo '${{ github.event.comment.body }}' > /tmp/raw_comment.txt + RAW_COMMENT_BODY=$(cat /tmp/raw_comment.txt) + COMMENT_CONTENT=$(echo "$RAW_COMMENT_BODY" | sed 's|^/aider||' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + + if [[ -z "$COMMENT_CONTENT" ]]; then + echo "::error::Comment with /aider provided, but no instruction found after it. Cannot proceed." + printf "Error: /aider command found but no instruction followed." > "$PROMPT_FILE_PATH" + exit 1 + else + echo "Using comment content as prompt." + printf '%s' "$COMMENT_CONTENT" > "$PROMPT_FILE_PATH" + fi + fi + else + echo "This is a comment on a regular issue" + + # Fetch the issue details + ISSUE_NUMBER="${{ github.event.issue.number }}" + ISSUE_DETAILS=$(gh issue view $ISSUE_NUMBER --json title,body --repo $GITHUB_REPOSITORY) + ISSUE_TITLE=$(echo "$ISSUE_DETAILS" | jq -r .title) + ISSUE_BODY=$(echo "$ISSUE_DETAILS" | jq -r .body) + + # Store raw comment body in a file first to avoid shell interpretation issues + echo '${{ github.event.comment.body }}' > /tmp/raw_comment.txt + # Extract the command part safely + RAW_COMMENT_BODY=$(cat /tmp/raw_comment.txt) + # Remove the /aider prefix and trim whitespace + COMMENT_CONTENT=$(echo "$RAW_COMMENT_BODY" | sed 's|^/aider||' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + + if [[ -z "$COMMENT_CONTENT" ]]; then + echo "::error::Comment with /aider provided, but no instruction found after it. Cannot proceed." + printf "Error: /aider command found but no instruction followed." > "$PROMPT_FILE_PATH" + exit 1 + else + echo "Sending issue content and issue comment to external API…" + + ISSUE_TITLE_Q=$(printf '%q' "$ISSUE_TITLE") + ISSUE_BODY_Q=$(printf '%q' "$ISSUE_BODY") + COMMENT_CONTENT_Q=$(printf '%q' "$COMMENT_CONTENT") + + JSON_PAYLOAD=$(jq -n \ + --arg title "$ISSUE_TITLE_Q" \ + --arg body "$ISSUE_BODY_Q" \ + --arg comment "$COMMENT_CONTENT_Q" \ + '{"body":{"issue_title":$title,"issue_body":$body,"issue_comment":$comment}}') + + API_RESULT=$(curl -s -w "\n%{http_code}" \ + -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run_wait_result/p/f/ai/quiet_script" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $WINDMILL_TOKEN" \ + --data-binary "$JSON_PAYLOAD" \ + --max-time 90) + + HTTP_CODE=$(echo "$API_RESULT" | tail -n1) + BODY=$(echo "$API_RESULT" | sed '$d') + + echo "$BODY" > /tmp/api_response.txt + + BASE_PROMPT="Try to fix the following issue based on the instruction given by the user. The issue is prepended with the word ISSUE. The instruction is prepended with the word INSTRUCTION. The issue and instruction are separated by a blank line." + if [[ "$HTTP_CODE" -eq 200 ]]; then + PROCESSED_ISSUE_PROMPT=$(jq -r '.effective_body // empty' /tmp/api_response.txt) + if [[ -z "$PROCESSED_ISSUE_PROMPT" || "$PROCESSED_ISSUE_PROMPT" == "null" ]]; then + PROCESSED_ISSUE_PROMPT="" + fi + printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \ + "$BASE_PROMPT" "$PROCESSED_ISSUE_PROMPT" "$COMMENT_CONTENT" > "$PROMPT_FILE_PATH" + else + echo "::warning::API call failed (HTTP $HTTP_CODE). Using PR comment with issue context." + printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \ + "$BASE_PROMPT" "$ISSUE_BODY_Q" "$COMMENT_CONTENT" > "$PROMPT_FILE_PATH" + fi + + rm -f /tmp/api_response.txt + fi + fi + echo "Prompt determined and written to $PROMPT_FILE_PATH" + echo "PROMPT_FILE_PATH=$PROMPT_FILE_PATH" >> $GITHUB_OUTPUT + + - name: Probe Chat for Relevant Files + id: probe_files + env: + PROMPT_CONTENT_FILE: ${{ steps.determine_prompt.outputs.PROMPT_FILE_PATH }} + run: | + echo "Running probe-chat to find relevant files..." + if [[ ! -f "$PROMPT_CONTENT_FILE" ]]; then + echo "::error::Prompt file $PROMPT_CONTENT_FILE not found!" + exit 1 + fi + PROMPT_CONTENT=$(cat "$PROMPT_CONTENT_FILE") + if [ -z "$PROMPT_CONTENT" ]; then + echo "::error::Prompt content is empty!" + exit 1 + fi + + PROMPT_ESCAPED=$(jq -Rs . <<< "$PROMPT_CONTENT") + + MESSAGE_FOR_PROBE=$(jq -n --arg prompt_escaped "$PROMPT_ESCAPED" \ + '{ "message": "I'\''m giving you a request that needs to be implemented. Your role is ONLY to give me the files that are relevant to the request and nothing else. The request is prepended with the word REQUEST.\\nREQUEST: \($prompt_escaped). Give me all the files relevant to this request. Your output MUST be a single json array that can be parsed with programatic json parsing, with the relevant files. Files can be rust or typescript or javascript files. DO NOT INCLUDE ANY OTHER TEXT IN YOUR OUTPUT. ONLY THE JSON ARRAY. Example of output: [\"file1.py\", \"file2.py\"]" }' | jq -r .message) + + set -o pipefail + PROBE_OUTPUT=$(npx --yes @buger/probe-chat@latest --max-iterations 50 --model-name gemini-2.5-pro-preview-05-06 --message "$MESSAGE_FOR_PROBE" 2>&1) || { + echo "::error::probe-chat command failed. Output:" + echo "$PROBE_OUTPUT" + exit 1 + } + set +o pipefail + echo "Probe-chat raw output:" + echo "$PROBE_OUTPUT" + + JSON_FILES=$(echo "$PROBE_OUTPUT" | sed -n '/^\s*\[/,$p' | sed '/^\s*\]/q') + echo "Extracted JSON block:" + echo "$JSON_FILES" + + FILES_LIST=$(echo "$JSON_FILES" | jq -e -r '[.[] | select(type == "string" and . != "" and . != null and (endswith("/") | not))] | map(@sh) | join(" ")' || echo "") + + if [[ -z "$FILES_LIST" ]]; then + echo "::warning::probe-chat did not identify any relevant files." + exit 1 + fi + + echo "Formatted files list for aider: $FILES_LIST" + echo "FILES_TO_EDIT=$FILES_LIST" >> $GITHUB_ENV + + - name: Run Aider with external prompt + run: | + echo "Files identified by probe-chat: ${{ env.FILES_TO_EDIT }}" + aider \ + --read .cursor/rules/rust-best-practices.mdc \ + --read .cursor/rules/svelte5-best-practices.mdc \ + ${{ env.FILES_TO_EDIT }} \ + --model gemini/gemini-2.5-pro-preview-05-06 \ + --message-file .github/aider/issue-prompt.txt \ + --yes \ + --no-check-update \ + --auto-commits \ + --no-analytics \ + --no-gitignore \ + | tee .github/aider/aider-output.txt || true + echo "Aider command completed. Output saved to .github/aider/aider-output.txt" + + - name: Clean up prompt file + if: always() + run: rm -f .github/aider/issue-prompt.txt + + - name: Commit and Push Changes + id: commit_and_push + if: ${{ success() }} + run: | + if [[ -z "${{ github.event.issue.pull_request }}" ]]; then + BRANCH_NAME="aider-fix-issue-${{ github.event.issue.number }}" + + # Check if branch exists remotely + if git ls-remote --heads origin $BRANCH_NAME | grep -q $BRANCH_NAME; then + echo "Branch $BRANCH_NAME already exists remotely, fetching it" + git fetch origin $BRANCH_NAME + git checkout $BRANCH_NAME + git pull origin $BRANCH_NAME + else + echo "Creating new branch $BRANCH_NAME" + git checkout -b $BRANCH_NAME + fi + + echo "Created/checked out branch $BRANCH_NAME for issue #${{ github.event.issue.number }}" + git push origin $BRANCH_NAME + echo "Pushed to branch $BRANCH_NAME" + echo "PR_BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT + echo "CHANGES_APPLIED_MESSAGE=Aider changes pushed to branch $BRANCH_NAME." >> $GITHUB_OUTPUT + else + CURRENT_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) + echo "Attempting to push changes to PR branch $CURRENT_BRANCH_NAME for PR #${{ github.event.issue.number }}" + if git push origin $CURRENT_BRANCH_NAME; then + echo "Push to $CURRENT_BRANCH_NAME successful (or no new changes to push)." + echo "CHANGES_APPLIED_MESSAGE=Aider changes (if any) pushed to PR branch $CURRENT_BRANCH_NAME." >> $GITHUB_OUTPUT + echo "PR_BRANCH_NAME=$CURRENT_BRANCH_NAME" >> $GITHUB_OUTPUT + else + echo "::warning::Push to PR branch $CURRENT_BRANCH_NAME failed." + echo "CHANGES_APPLIED_MESSAGE=Aider ran, but failed to push changes to PR branch $CURRENT_BRANCH_NAME." >> $GITHUB_OUTPUT + fi + fi + + - name: Create Pull Request + if: success() && github.event_name == 'issue_comment' && !github.event.issue.pull_request && steps.commit_and_push.outputs.PR_BRANCH_NAME != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_BRANCH: ${{ steps.commit_and_push.outputs.PR_BRANCH_NAME }} + ISSUE_NUM: ${{ github.event.issue.number }} + run: | + # Create PR description in a temporary file to avoid command line length limits + cat > /tmp/pr-description.md << EOL + This PR was created automatically by Aider to fix issue #${ISSUE_NUM}. + + ## Aider Output + \`\`\` + $(cat .github/aider/aider-output.txt || echo "No output available") + \`\`\` + EOL + + # Create PR using the file for the body content + gh pr create \ + --title "[Aider PR] Add fixes for issue #${ISSUE_NUM}" \ + --body-file /tmp/pr-description.md \ + --head "$PR_BRANCH" \ + --base main diff --git a/.github/workflows/pr-opened.yml b/.github/workflows/pr-opened.yml new file mode 100644 index 0000000000..8e098f1e3c --- /dev/null +++ b/.github/workflows/pr-opened.yml @@ -0,0 +1,34 @@ +name: "Notify Discord on New PR (with Thread)" + +on: + pull_request: + types: + - opened + - ready_for_review + +jobs: + discord_notification: + # still guard out any drafts (just in case) + if: github.event.pull_request.draft == false + runs-on: ubicloud-standard-2 + steps: + - name: Send Discord notification and start a thread + env: + WEBHOOK_URL: ${{ secrets.DISCORD_PR_REVIEWS_WEBHOOK }} + PR_TITLE: ${{ github.event.pull_request.title }} + PR_URL: ${{ github.event.pull_request.html_url }} + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + run: | + payload=$(jq -n \ + --arg content "${PR_URL}" \ + --arg thread "$PR_TITLE by \`${PR_AUTHOR}\`" \ + '{ + content: $content, + thread_name: $thread, + auto_archive_duration: 10080 + }' + ) + curl -H "Content-Type: application/json" \ + -X POST \ + --data "$payload" \ + "$WEBHOOK_URL" diff --git a/.gitignore b/.gitignore index b157a1679a..5136fb8aca 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ CaddyfileRemoteMalo .vscode .dev-docker-wrapper* backend/.minio-data +.aider* +!.aiderignore \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 2814c7bb72..f612d37c90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [1.491.1](https://github.com/windmill-labs/windmill/compare/v1.491.0...v1.491.1) (2025-05-15) + + +### Bug Fixes + +* avoid deadlocks in sending completed job to result processors ([#5742](https://github.com/windmill-labs/windmill/issues/5742)) ([e87d4f3](https://github.com/windmill-labs/windmill/commit/e87d4f3c1afb4ad356b326b7600c89e6c7803eff)) + +## [1.491.0](https://github.com/windmill-labs/windmill/compare/v1.490.0...v1.491.0) (2025-05-14) + + +### Features + +* Microsoft Teams approvals ([#5734](https://github.com/windmill-labs/windmill/issues/5734)) ([039f3e0](https://github.com/windmill-labs/windmill/commit/039f3e02268f2acda48abea420479216970e58e7)) +* sql jobs outputting to s3 + streaming for high-number of rows ([#5704](https://github.com/windmill-labs/windmill/issues/5704)) ([c7886ea](https://github.com/windmill-labs/windmill/commit/c7886ea07ae44af56f1467288b2d73ff2ae27964)) + + +### Bug Fixes + +* add missing run job transaction drop ([#5730](https://github.com/windmill-labs/windmill/issues/5730)) ([318def9](https://github.com/windmill-labs/windmill/commit/318def976cf0e4d5c32d01ac611a89e0a6425368)) +* add support for log compaction on docker jobs ([#5732](https://github.com/windmill-labs/windmill/issues/5732)) ([d35a7d2](https://github.com/windmill-labs/windmill/commit/d35a7d22f960f485889e22de48e8de8557069cb7)) +* Ansible lockfile back compatibility issue ([#5731](https://github.com/windmill-labs/windmill/issues/5731)) ([f73c90c](https://github.com/windmill-labs/windmill/commit/f73c90c7518569204b298b916d0fc298932d3cf0)) +* trigger event support for webhook get endpoints ([#5728](https://github.com/windmill-labs/windmill/issues/5728)) ([76258b7](https://github.com/windmill-labs/windmill/commit/76258b7b1af1313f694731d77f3fa6994e9ded70)) + ## [1.490.0](https://github.com/windmill-labs/windmill/compare/v1.489.0...v1.490.0) (2025-05-12) diff --git a/backend/.sqlx/query-3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e.json b/backend/.sqlx/query-3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e.json new file mode 100644 index 0000000000..9f5e2e9d8f --- /dev/null +++ b/backend/.sqlx/query-3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e.json @@ -0,0 +1,91 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH job_info AS (\n -- Query for Teams (running jobs)\n SELECT\n parent.job_kind AS \"job_kind!: JobKind\",\n parent.script_hash AS \"script_hash: ScriptHash\",\n parent.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n child.parent_job AS \"parent_job: Uuid\",\n parent.created_at AS \"created_at!: chrono::NaiveDateTime\",\n parent.created_by AS \"created_by!\",\n parent.script_path,\n parent.args AS \"args: sqlx::types::Json>\"\n FROM v2_as_queue child\n JOIN v2_as_queue parent ON parent.id = child.parent_job\n WHERE child.id = $1 AND child.workspace_id = $2\n UNION ALL\n -- Query for Slack (completed jobs)\n SELECT\n v2_as_queue.job_kind AS \"job_kind!: JobKind\",\n v2_as_queue.script_hash AS \"script_hash: ScriptHash\",\n v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n v2_as_completed_job.parent_job AS \"parent_job: Uuid\",\n v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\",\n v2_as_completed_job.created_by AS \"created_by!\",\n v2_as_queue.script_path,\n v2_as_queue.args AS \"args: sqlx::types::Json>\"\n FROM v2_as_queue\n JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id\n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2\n )\n SELECT * FROM job_info LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlescriptflow", + "flowscript", + "flownode", + "appscript" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "script_hash: ScriptHash", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "raw_flow: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "parent_job: Uuid", + "type_info": "Uuid" + }, + { + "ordinal": 4, + "name": "created_at!: chrono::NaiveDateTime", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "args: sqlx::types::Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e" +} diff --git a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json index 54e94cfb8f..99269c9851 100644 --- a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json +++ b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json @@ -18,8 +18,8 @@ "Left": [] }, "nullable": [ - false, - true + true, + false ] }, "hash": "b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76" diff --git a/backend/.sqlx/query-f0fdeb7aea3e71099e7db0f4343bbd7ec86610ddc8589bf5b606fab0947c8b75.json b/backend/.sqlx/query-f0fdeb7aea3e71099e7db0f4343bbd7ec86610ddc8589bf5b606fab0947c8b75.json deleted file mode 100644 index 26111685ba..0000000000 --- a/backend/.sqlx/query-f0fdeb7aea3e71099e7db0f4343bbd7ec86610ddc8589bf5b606fab0947c8b75.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n v2_as_queue.job_kind AS \"job_kind!: JobKind\",\n v2_as_queue.script_hash AS \"script_hash: ScriptHash\",\n v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n v2_as_completed_job.parent_job AS \"parent_job: Uuid\",\n v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\",\n v2_as_completed_job.created_by AS \"created_by!\",\n v2_as_queue.script_path,\n v2_as_queue.args AS \"args: sqlx::types::Json>\"\n FROM v2_as_queue\n JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id\n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2\n LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "job_kind!: JobKind", - "type_info": { - "Custom": { - "name": "job_kind", - "kind": { - "Enum": [ - "script", - "preview", - "flow", - "dependencies", - "flowpreview", - "script_hub", - "identity", - "flowdependencies", - "http", - "graphql", - "postgresql", - "noop", - "appdependencies", - "deploymentcallback", - "singlescriptflow", - "flowscript", - "flownode", - "appscript" - ] - } - } - } - }, - { - "ordinal": 1, - "name": "script_hash: ScriptHash", - "type_info": "Int8" - }, - { - "ordinal": 2, - "name": "raw_flow: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 3, - "name": "parent_job: Uuid", - "type_info": "Uuid" - }, - { - "ordinal": 4, - "name": "created_at!: chrono::NaiveDateTime", - "type_info": "Timestamptz" - }, - { - "ordinal": 5, - "name": "created_by!", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "args: sqlx::types::Json>", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - true, - true, - true, - true, - true, - true, - true, - true - ] - }, - "hash": "f0fdeb7aea3e71099e7db0f4343bbd7ec86610ddc8589bf5b606fab0947c8b75" -} diff --git a/backend/.sqlx/query-fec6d5674dc6b5a6a0ece419c40508835affcb7679a48f2a443777e829bd1e74.json b/backend/.sqlx/query-fec6d5674dc6b5a6a0ece419c40508835affcb7679a48f2a443777e829bd1e74.json index 649ab85650..de2e819af2 100644 --- a/backend/.sqlx/query-fec6d5674dc6b5a6a0ece419c40508835affcb7679a48f2a443777e829bd1e74.json +++ b/backend/.sqlx/query-fec6d5674dc6b5a6a0ece419c40508835affcb7679a48f2a443777e829bd1e74.json @@ -41,11 +41,11 @@ ] }, "nullable": [ - false, - false, - false, - false, - false, + true, + true, + true, + true, + true, true ] }, diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 97203b8559..5128902eff 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -288,9 +288,9 @@ dependencies = [ [[package]] name = "arrow" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3095aaf545942ff5abd46654534f15b03a90fba78299d661e045e5d587222f0d" +checksum = "b1bb018b6960c87fd9d025009820406f74e83281185a8bdcb44880d2aa5c9a87" dependencies = [ "arrow-arith", "arrow-array", @@ -309,9 +309,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00752064ff47cee746e816ddb8450520c3a52cbad1e256f6fa861a35f86c45e7" +checksum = "44de76b51473aa888ecd6ad93ceb262fb8d40d1f1154a4df2f069b3590aa7575" dependencies = [ "arrow-array", "arrow-buffer", @@ -323,9 +323,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cebfe926794fbc1f49ddd0cdaf898956ca9f6e79541efce62dabccfd81380472" +checksum = "29ed77e22744475a9a53d00026cf8e166fe73cf42d89c4c4ae63607ee1cfcc3f" dependencies = [ "ahash 0.8.12", "arrow-buffer", @@ -340,9 +340,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0303c7ec4cf1a2c60310fc4d6bbc3350cd051a17bf9e9c0a8e47b4db79277824" +checksum = "b0391c96eb58bf7389171d1e103112d3fc3e5625ca6b372d606f2688f1ea4cce" dependencies = [ "bytes", "half", @@ -351,9 +351,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335f769c5a218ea823d3760a743feba1ef7857cba114c01399a891c2fff34285" +checksum = "f39e1d774ece9292697fcbe06b5584401b26bd34be1bec25c33edae65c2420ff" dependencies = [ "arrow-array", "arrow-buffer", @@ -372,9 +372,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "510db7dfbb4d5761826516cc611d97b3a68835d0ece95b034a052601109c0b1b" +checksum = "9055c972a07bf12c2a827debfd34f88d3b93da1941d36e1d9fee85eebe38a12a" dependencies = [ "arrow-array", "arrow-cast", @@ -388,9 +388,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8affacf3351a24039ea24adab06f316ded523b6f8c3dbe28fbac5f18743451b" +checksum = "cf75ac27a08c7f48b88e5c923f267e980f27070147ab74615ad85b5c5f90473d" dependencies = [ "arrow-buffer", "arrow-schema", @@ -400,9 +400,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69880a9e6934d9cba2b8630dd08a3463a91db8693b16b499d54026b6137af284" +checksum = "a222f0d93772bd058d1268f4c28ea421a603d66f7979479048c429292fac7b2e" dependencies = [ "arrow-array", "arrow-buffer", @@ -414,9 +414,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8dafd17a05449e31e0114d740530e0ada7379d7cb9c338fd65b09a8130960b0" +checksum = "9085342bbca0f75e8cb70513c0807cc7351f1fbf5cb98192a67d5e3044acb033" dependencies = [ "arrow-array", "arrow-buffer", @@ -436,9 +436,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "895644523af4e17502d42c3cb6b27cb820f0cb77954c22d75c23a85247c849e1" +checksum = "ab2f1065a5cad7b9efa9e22ce5747ce826aa3855766755d4904535123ef431e7" dependencies = [ "arrow-array", "arrow-buffer", @@ -449,9 +449,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9be8a2a4e5e7d9c822b2b8095ecd77010576d824f654d347817640acfc97d229" +checksum = "3703a0e3e92d23c3f756df73d2dc9476873f873a76ae63ef9d3de17fda83b2d8" dependencies = [ "arrow-array", "arrow-buffer", @@ -462,15 +462,15 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7450c76ab7c5a6805be3440dc2e2096010da58f7cab301fdc996a4ee3ee74e49" +checksum = "73a47aa0c771b5381de2b7f16998d351a6f4eb839f1e13d48353e17e873d969b" [[package]] name = "arrow-select" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa5f5a93c75f46ef48e4001535e7b6c922eeb0aa20b73cf58d09e13d057490d8" +checksum = "24b7b85575702b23b85272b01bc1c25a01c9b9852305e5d0078c79ba25d995d4" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -482,9 +482,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7005d858d84b56428ba2a98a107fe88c0132c61793cf6b8232a1f9bfc0452b" +checksum = "9260fddf1cdf2799ace2b4c2fc0356a9789fa7551e0953e35435536fecefebbd" dependencies = [ "arrow-array", "arrow-buffer", @@ -1303,7 +1303,7 @@ version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "cexpr", "clang-sys", "itertools 0.12.1", @@ -1326,7 +1326,7 @@ version = "0.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1378,9 +1378,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" dependencies = [ "serde", ] @@ -1563,7 +1563,7 @@ checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor", + "brotli-decompressor 4.0.3", ] [[package]] @@ -1574,7 +1574,18 @@ checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor", + "brotli-decompressor 4.0.3", +] + +[[package]] +name = "brotli" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor 5.0.0", ] [[package]] @@ -1587,6 +1598,16 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bstr" version = "1.12.0" @@ -2417,7 +2438,7 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b28bfe653d79bd16c77f659305b195b82bb5ce0c0eb2a4846b82ddbd77586813" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "libloading 0.8.7", "winapi", ] @@ -5054,7 +5075,7 @@ version = "25.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "rustc_version 0.4.1", ] @@ -5600,15 +5621,16 @@ dependencies = [ [[package]] name = "generator" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6bd114ceda131d3b1d665eba35788690ad37f5916457286b32ab6fd3c438dd" +checksum = "d18470a76cb7f8ff746cf1f7470914f900252ec36bbc40b569d74b1258446827" dependencies = [ + "cc", "cfg-if", "libc", "log", "rustversion", - "windows 0.58.0", + "windows 0.61.1", ] [[package]] @@ -5873,7 +5895,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbcd2dba93594b227a1f57ee09b8b9da8892c34d55aa332e034a228d0fe6a171" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "gpu-alloc-types", ] @@ -5883,7 +5905,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98ff03b468aa837d70984d55f5d3f846f6ec31fe34bbb97c4f85219caeee1ca4" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] @@ -5892,7 +5914,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf29e94d6d243368b7a56caa16bc213e4f9f8ed38c4d9557069527b5d5281ca" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "gpu-descriptor-types", "hashbrown 0.15.3", ] @@ -5903,7 +5925,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdf242682df893b86f33a73828fb09ca4b2d3bb6cc95249707fc684d27484b91" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] @@ -7183,7 +7205,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "libc", "redox_syscall 0.5.12", ] @@ -7612,7 +7634,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5637e166ea14be6063a3f8ba5ccb9a4159df7d8f6d61c02fc3d480b1f90dcfcb" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "block", "core-graphics-types", "foreign-types 0.5.0", @@ -7853,7 +7875,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e0ec195e788c95f36b7cf88127d538465fc2f7773e6e47af01834738eab0aee" dependencies = [ "base64 0.22.1", - "bitflags 2.9.0", + "bitflags 2.9.1", "btoi", "byteorder", "bytes", @@ -7882,7 +7904,7 @@ checksum = "e536ae46fcab0876853bd4a632ede5df4b1c2527a58f6c5a4150fe86be858231" dependencies = [ "arrayvec", "bit-set 0.5.3", - "bitflags 2.9.0", + "bitflags 2.9.1", "codespan-reporting", "hexf-parse", "indexmap 2.9.0", @@ -7974,7 +7996,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "cfg-if", "libc", ] @@ -7985,7 +8007,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "cfg-if", "cfg_aliases 0.2.1", "libc", @@ -8058,7 +8080,7 @@ version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "crossbeam-channel", "filetime", "fsevent-sys", @@ -8572,7 +8594,7 @@ version = "0.10.72" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "cfg-if", "foreign-types 0.3.2", "libc", @@ -8911,9 +8933,9 @@ dependencies = [ [[package]] name = "parquet" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd31a8290ac5b19f09ad77ee7a1e6a541f1be7674ad410547d5f1eef6eef4a9c" +checksum = "be7b2d778f6b841d37083ebdf32e33a524acde1266b5884a8ca29bf00dfa1231" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -8924,7 +8946,7 @@ dependencies = [ "arrow-schema", "arrow-select", "base64 0.22.1", - "brotli 7.0.0", + "brotli 8.0.1", "bytes", "chrono", "flate2", @@ -9444,7 +9466,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "chrono", "flate2", "hex", @@ -9458,7 +9480,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "chrono", "hex", ] @@ -9584,7 +9606,7 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "getopts", "memchr", "unicase", @@ -9851,7 +9873,7 @@ version = "11.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] @@ -9973,7 +9995,7 @@ version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] @@ -10206,9 +10228,9 @@ dependencies = [ [[package]] name = "resolv-conf" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7c8f7f733062b66dc1c63f9db168ac0b97a9210e247fa90fdc9ad08f51b302" +checksum = "95325155c684b1c89f7765e30bc1c42e4a6da51ca513615660cb8a62ef9a88e3" [[package]] name = "retry-policies" @@ -10337,7 +10359,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" dependencies = [ "base64 0.21.7", - "bitflags 2.9.0", + "bitflags 2.9.1", "serde", "serde_derive", ] @@ -10388,7 +10410,7 @@ version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "fallible-iterator 0.3.0", "fallible-streaming-iterator", "hashlink 0.9.1", @@ -10509,7 +10531,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -10522,7 +10544,7 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "errno", "libc", "linux-raw-sys 0.9.4", @@ -10750,7 +10772,7 @@ version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02a2d683a4ac90aeef5b1013933f6d977bd37d51ff3f4dad829d4931a7e6be86" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "cfg-if", "clipboard-win", "fd-lock", @@ -10943,7 +10965,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -10956,7 +10978,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "core-foundation 0.10.0", "core-foundation-sys", "libc", @@ -11512,7 +11534,7 @@ version = "0.3.0+sdk-1.3.268.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eda41003dc44290527a59b13432d4a0379379fa074b70174882adfbdfd917844" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] @@ -11675,7 +11697,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.9.0", + "bitflags 2.9.1", "byteorder", "bytes", "chrono", @@ -11720,7 +11742,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.9.0", + "bitflags 2.9.1", "byteorder", "chrono", "crc", @@ -12000,7 +12022,7 @@ version = "0.118.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6f866d12e4d519052b92a0a86d1ac7ff17570da1272ca0c89b3d6f802cd79df" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "is-macro", "num-bigint", "phf", @@ -12086,7 +12108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65f21494e75d0bd8ef42010b47cabab9caaed8f2207570e809f6f4eb51a710d1" dependencies = [ "better_scoped_tls", - "bitflags 2.9.0", + "bitflags 2.9.1", "indexmap 2.9.0", "once_cell", "phf", @@ -12354,7 +12376,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "byteorder", "enum-as-inner", "libc", @@ -12368,7 +12390,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "byteorder", "enum-as-inner", "libc", @@ -12407,7 +12429,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "core-foundation 0.9.4", "system-configuration-sys 0.6.0", ] @@ -13290,7 +13312,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fdb0c213ca27a9f57ab69ddb290fd80d970922355b83ae380b395d3986b8a2e" dependencies = [ "async-compression", - "bitflags 2.9.0", + "bitflags 2.9.1", "bytes", "futures-core", "http 1.3.1", @@ -13923,7 +13945,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a511192602f7b435b0a241c1947aa743eb7717f20a9195f4b5e8ed1952e01db1" dependencies = [ "bindgen 0.70.1", - "bitflags 2.9.0", + "bitflags 2.9.1", "fslock", "gzip-header", "home", @@ -13939,7 +13961,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97599c400fc79925922b58303e98fcb8fa88f573379a08ddb652e72cbd2e70f6" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "encoding_rs", "indexmap 2.9.0", "num-bigint", @@ -14230,7 +14252,7 @@ checksum = "d50819ab545b867d8a454d1d756b90cd5f15da1f2943334ca314af10583c9d39" dependencies = [ "arrayvec", "bit-vec 0.6.3", - "bitflags 2.9.0", + "bitflags 2.9.1", "cfg_aliases 0.1.1", "codespan-reporting", "document-features", @@ -14261,7 +14283,7 @@ dependencies = [ "arrayvec", "ash", "bit-set 0.5.3", - "bitflags 2.9.0", + "bitflags 2.9.1", "block", "cfg_aliases 0.1.1", "core-graphics-types", @@ -14299,7 +14321,7 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1353d9a46bff7f955a680577f34c69122628cc2076e1d6f3a9be6ef00ae793ef" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "js-sys", "serde", "web-sys", @@ -14379,7 +14401,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "axum", @@ -14428,7 +14450,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "argon2", @@ -14537,7 +14559,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.490.0" +version = "1.491.1" dependencies = [ "base64 0.22.1", "chrono", @@ -14552,7 +14574,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.490.0" +version = "1.491.1" dependencies = [ "chrono", "serde", @@ -14565,7 +14587,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "serde", @@ -14579,7 +14601,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "async-stream", @@ -14649,7 +14671,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.490.0" +version = "1.491.1" dependencies = [ "regex", "serde", @@ -14663,7 +14685,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "bytes", @@ -14686,7 +14708,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.490.0" +version = "1.491.1" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14698,7 +14720,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.490.0" +version = "1.491.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -14707,7 +14729,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "lazy_static", @@ -14719,7 +14741,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "serde_json", @@ -14731,7 +14753,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "gosyn", @@ -14743,7 +14765,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "lazy_static", @@ -14755,7 +14777,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "serde_json", @@ -14767,7 +14789,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "nu-parser", @@ -14778,7 +14800,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14789,7 +14811,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14800,7 +14822,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "async-recursion", @@ -14820,7 +14842,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -14837,7 +14859,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "lazy_static", @@ -14849,7 +14871,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "lazy_static", @@ -14867,7 +14889,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "getrandom 0.2.16", @@ -14891,7 +14913,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "serde_json", @@ -14901,7 +14923,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "async-recursion", @@ -14934,7 +14956,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.490.0" +version = "1.491.1" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -14944,7 +14966,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.490.0" +version = "1.491.1" dependencies = [ "anyhow", "async-recursion", @@ -15005,6 +15027,7 @@ dependencies = [ "tiberius", "tokio", "tokio-postgres 0.7.13", + "tokio-stream", "tokio-util", "tracing", "url", @@ -15055,12 +15078,24 @@ dependencies = [ [[package]] name = "windows" -version = "0.58.0" +version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +checksum = "c5ee8f3d025738cb02bad7868bbb5f8a6327501e870bf51f1b455b0a2454a419" dependencies = [ - "windows-core 0.58.0", - "windows-targets 0.52.6", + "windows-collections", + "windows-core 0.61.0", + "windows-future", + "windows-link", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core 0.61.0", ] [[package]] @@ -15087,19 +15122,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-core" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" -dependencies = [ - "windows-implement 0.58.0", - "windows-interface 0.58.0", - "windows-result 0.2.0", - "windows-strings 0.1.0", - "windows-targets 0.52.6", -] - [[package]] name = "windows-core" version = "0.61.0" @@ -15113,6 +15135,16 @@ dependencies = [ "windows-strings 0.4.0", ] +[[package]] +name = "windows-future" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a1d6bbefcb7b60acd19828e1bc965da6fcf18a7e39490c5f8be71e54a19ba32" +dependencies = [ + "windows-core 0.61.0", + "windows-link", +] + [[package]] name = "windows-implement" version = "0.56.0" @@ -15135,17 +15167,6 @@ dependencies = [ "syn 2.0.101", ] -[[package]] -name = "windows-implement" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.101", -] - [[package]] name = "windows-implement" version = "0.60.0" @@ -15179,17 +15200,6 @@ dependencies = [ "syn 2.0.101", ] -[[package]] -name = "windows-interface" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.101", -] - [[package]] name = "windows-interface" version = "0.59.1" @@ -15207,6 +15217,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core 0.61.0", + "windows-link", +] + [[package]] name = "windows-registry" version = "0.4.0" @@ -15227,15 +15247,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-result" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-result" version = "0.3.2" @@ -15245,16 +15256,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-strings" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" -dependencies = [ - "windows-result 0.2.0", - "windows-targets 0.52.6", -] - [[package]] name = "windows-strings" version = "0.3.1" @@ -15534,7 +15535,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index f2a92433ae..a8705c88f7 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.490.0" +version = "1.491.1" authors.workspace = true edition.workspace = true @@ -32,7 +32,7 @@ members = [ ] [workspace.package] -version = "1.490.0" +version = "1.491.1" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 9600e384bf..46189459c0 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -4dc1f25f4fcc013334d4cc1d07cbe60a22b56d1f \ No newline at end of file +b77d145e278de3bd4079e3228733df24cc4c0070 \ No newline at end of file diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 544b1348d1..b1a90bafb1 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.490.0 + version: 1.491.1 title: Windmill API contact: @@ -7899,6 +7899,52 @@ paths: "200": description: Interactive slack approval message sent successfully + /w/{workspace}/jobs/teams_approval/{id}: + get: + summary: generate interactive teams approval for suspended job + operationId: getTeamsApprovalPayload + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/JobId" + - name: approver + in: query + schema: + type: string + - name: message + in: query + schema: + type: string + - name: team_name + in: query + required: true + schema: + type: string + - name: channel_name + in: query + required: true + schema: + type: string + - name: flow_step_id + in: query + required: true + schema: + type: string + - name: default_args_json + in: query + required: false + schema: + type: string + - name: dynamic_enums_json + in: query + required: false + schema: + type: string + responses: + "200": + description: Interactive slack approval message sent successfully + /w/{workspace}/jobs_u/resume/{id}/{resume_id}/{signature}: get: summary: resume a job for a suspended flow diff --git a/backend/windmill-api/src/agent_workers_ee.rs b/backend/windmill-api/src/agent_workers_ee.rs index 4ffc0550ef..1b5a210d13 100644 --- a/backend/windmill-api/src/agent_workers_ee.rs +++ b/backend/windmill-api/src/agent_workers_ee.rs @@ -28,7 +28,7 @@ pub fn workspaced_service( use windmill_worker::JobCompletedSender; let (job_completed_tx, _job_completed_rx) = - JobCompletedSender::new(&Connection::Sql(db.clone()), 100); + JobCompletedSender::new(&Connection::Sql(db.clone()), 10); let router = Router::new(); diff --git a/backend/windmill-api/src/approvals.rs b/backend/windmill-api/src/approvals.rs new file mode 100644 index 0000000000..f4ee9dd843 --- /dev/null +++ b/backend/windmill-api/src/approvals.rs @@ -0,0 +1,292 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use uuid::Uuid; +use std::str::FromStr; +use regex::Regex; +use serde_json::Value; +use crate::db::{ApiAuthed, DB}; +use crate::jobs::{cancel_suspended_job, resume_suspended_job, QueryApprover, QueryOrBody, ResumeUrls, get_resume_urls_internal}; +use axum::{extract::{Path, Query}, Extension}; +use windmill_common::error::Error; +use windmill_common::cache; +use windmill_common::jobs::JobKind; +use windmill_common::scripts::ScriptHash; +use serde_json::value::RawValue; + +#[derive(Debug, Deserialize, Serialize)] +pub struct ResumeSchema { + pub schema: Schema, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct Schema { + pub order: Vec, + pub required: Vec, + pub properties: HashMap, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +#[serde(rename_all = "lowercase")] +pub enum FieldType { + Boolean, + String, + Number, + Integer, + #[serde(other)] + Unknown, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct ResumeFormField { + pub r#type: FieldType, + pub format: Option, + pub default: Option, + pub description: Option, + pub title: Option, + pub r#enum: Option>, + #[serde(rename = "enumLabels")] + pub enum_labels: Option>, + pub nullable: Option, + pub placeholder: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct ResumeFormRow { + pub resume_form: Option, + pub hide_cancel: Option, +} + +#[derive(Deserialize)] +pub struct QueryMessage { + pub message: Option, +} + +#[derive(Deserialize)] +pub struct QueryFlowStepId { + pub flow_step_id: String, +} + +#[derive(Deserialize, Debug)] +pub struct QueryDefaultArgsJson { + pub default_args_json: Option, +} + +#[derive(Deserialize, Debug)] +pub struct QueryDynamicEnumJson { + pub dynamic_enums_json: Option, +} + +#[derive(Debug)] +pub struct ApprovalFormDetails { + pub message_str: String, + pub urls: ResumeUrls, + pub schema: Option, +} + +#[allow(dead_code)] +#[derive(Debug, Clone, Copy)] +pub enum MessageFormat { + Slack, + Teams, +} + +pub fn extract_w_id_from_resume_url(resume_url: &str) -> Result<&str, Error> { + let re = Regex::new(r"/api/w/(?P[^/]+)/jobs_u/(?Presume|cancel)/(?P[^/]+)/(?P[^/]+)/(?P[a-fA-F0-9]+)(?:\?approver=(?P[^&]+))?").unwrap(); + let captures = re.captures(resume_url).ok_or_else(|| { + tracing::error!("Resume URL does not match the pattern."); + Error::BadRequest("Invalid URL format.".to_string()) + })?; + Ok(captures.name("w_id").map_or("", |m| m.as_str())) +} + +pub async fn handle_resume_action( + authed: Option, + db: DB, + resume_url: &str, + form_data: Value, + action: &str, +) -> Result<(), Error> { + // Extract information from resume_url using regex + let re = Regex::new(r"/api/w/(?P[^/]+)/jobs_u/(?Presume|cancel)/(?P[^/]+)/(?P[^/]+)/(?P[a-fA-F0-9]+)(?:\?approver=(?P[^&]+))?").unwrap(); + let captures = re.captures(resume_url).ok_or_else(|| { + tracing::error!("Resume URL does not match the pattern."); + Error::BadRequest("Invalid URL format.".to_string()) + })?; + + let (w_id, job_id, resume_id, secret, approver) = ( + captures.name("w_id").map_or("", |m| m.as_str()), + captures.name("job_id").map_or("", |m| m.as_str()), + captures.name("resume_id").map_or("", |m| m.as_str()), + captures.name("secret").map_or("", |m| m.as_str()), + captures.name("approver").map(|m| m.as_str().to_string()), + ); + + let approver = QueryApprover { approver }; + + // Convert job_id and resume_id to appropriate types + let job_uuid = Uuid::from_str(job_id) + .map_err(|_| Error::BadRequest("Invalid job ID format.".to_string()))?; + + let resume_id_parsed = resume_id + .parse::() + .map_err(|_| Error::BadRequest("Invalid resume ID format.".to_string()))?; + + // Call the appropriate function based on the action + let res = if action == "resume" { + resume_suspended_job( + authed, + Extension(db.clone()), + Path(( + w_id.to_string(), + job_uuid, + resume_id_parsed, + secret.to_string(), + )), + Query(approver), + QueryOrBody(Some(form_data)), + ) + .await + } else { + cancel_suspended_job( + authed, + Extension(db.clone()), + Path(( + w_id.to_string(), + job_uuid, + resume_id_parsed, + secret.to_string(), + )), + Query(approver), + QueryOrBody(Some(form_data)), + ) + .await + }; + + tracing::debug!("Job action result: {:#?}", res); + res?; + + Ok(()) +} + +pub async fn get_approval_form_details( + db: DB, + w_id: &str, + job_id: Uuid, + flow_step_id: Option<&str>, + resume_id: u32, + approver: Option<&str>, + message: Option<&str>, + format: MessageFormat, +) -> Result { + let res = get_resume_urls_internal( + axum::Extension(db.clone()), + Path((w_id.to_string(), job_id, resume_id)), + Query(QueryApprover { approver: approver.map(|a| a.to_string()) }), + ) + .await?; + + let urls = res.0; + + tracing::debug!("Job ID: {:?}", job_id); + + // TODO: do we have a helper function for this? + let (job_kind, script_hash, raw_flow, parent_job_id, created_at, created_by, script_path, args) = sqlx::query!( + "WITH job_info AS ( + -- Query for Teams (running jobs) + SELECT + parent.job_kind AS \"job_kind!: JobKind\", + parent.script_hash AS \"script_hash: ScriptHash\", + parent.raw_flow AS \"raw_flow: sqlx::types::Json>\", + child.parent_job AS \"parent_job: Uuid\", + parent.created_at AS \"created_at!: chrono::NaiveDateTime\", + parent.created_by AS \"created_by!\", + parent.script_path, + parent.args AS \"args: sqlx::types::Json>\" + FROM v2_as_queue child + JOIN v2_as_queue parent ON parent.id = child.parent_job + WHERE child.id = $1 AND child.workspace_id = $2 + UNION ALL + -- Query for Slack (completed jobs) + SELECT + v2_as_queue.job_kind AS \"job_kind!: JobKind\", + v2_as_queue.script_hash AS \"script_hash: ScriptHash\", + v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json>\", + v2_as_completed_job.parent_job AS \"parent_job: Uuid\", + v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\", + v2_as_completed_job.created_by AS \"created_by!\", + v2_as_queue.script_path, + v2_as_queue.args AS \"args: sqlx::types::Json>\" + FROM v2_as_queue + JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id + WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 + ) + SELECT * FROM job_info LIMIT 1", + job_id, + &w_id + ) + .fetch_optional(&db) + .await + .map_err(|e| Error::BadRequest(e.to_string()))? + .ok_or_else(|| Error::BadRequest("This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string())) + .map(|r| (r.job_kind, r.script_hash, r.raw_flow, r.parent_job, r.created_at, r.created_by, r.script_path, r.args))?; + + let flow_data = match cache::job::fetch_flow(&db, job_kind, script_hash).await { + Ok(data) => data, + Err(_) => { + if let Some(parent_job_id) = parent_job_id.as_ref() { + cache::job::fetch_preview_flow(&db, parent_job_id, raw_flow).await? + } else { + return Err(Error::BadRequest( + "This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string(), + )); + } + } + }; + + let flow_value = &flow_data.flow; + let flow_step_id = flow_step_id.unwrap_or(""); + let module = flow_value.modules.iter().find(|m| m.id == flow_step_id); + + tracing::debug!("Module: {:#?}", module); + + let schema = module.and_then(|module| { + module.suspend.as_ref().map(|suspend| ResumeFormRow { + resume_form: suspend.resume_form.clone(), + hide_cancel: suspend.hide_cancel, + }) + }); + + let args_str = args.map_or("None".to_string(), |a| a.get().to_string()); + let parent_job_id_str = parent_job_id.map_or("None".to_string(), |id| id.to_string()); + let script_path_str = script_path.as_deref().unwrap_or("None"); + + let created_at_formatted = created_at.format("%Y-%m-%d %H:%M:%S").to_string(); + + let bold_format = match format { + MessageFormat::Slack => "*{}*", + MessageFormat::Teams => "**{}**", + }; + + let mut message_str = format!( + "A workflow has been suspended and is waiting for approval:\n\n\ + {}: {created_by}\n\n\ + {}: {created_at_formatted}\n\n\ + {}: {script_path_str}\n\n\ + {}: {args_str}\n\n\ + {}: {parent_job_id_str}\n\n", + bold_format.replace("{}", "Created by"), + bold_format.replace("{}", "Created at"), + bold_format.replace("{}", "Script path"), + bold_format.replace("{}", "Args"), + bold_format.replace("{}", "Flow ID") + ); + + // Append custom message if provided + if let Some(msg) = message { + message_str.push_str(msg); + } + + tracing::debug!("Schema: {:#?}", schema); + + Ok(ApprovalFormDetails { message_str, urls, schema }) +} diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 0c3840adc4..94b74ef9b9 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -381,6 +381,7 @@ async fn set_gcp_trigger_config( gcp_config.subscription_mode, gcp_config.create_update, false, + capture_config.is_flow, ) .await?; gcp_config.create_update = Some(config); diff --git a/backend/windmill-api/src/gcp_triggers_ee.rs b/backend/windmill-api/src/gcp_triggers_ee.rs index bdf98c9bb9..6e39cfe07c 100644 --- a/backend/windmill-api/src/gcp_triggers_ee.rs +++ b/backend/windmill-api/src/gcp_triggers_ee.rs @@ -86,6 +86,7 @@ pub async fn manage_google_subscription( _subscription_mode: SubscriptionMode, _create_update_config: Option, _trigger_mode: bool, + _is_flow: bool ) -> WindmillResult { Ok(CreateUpdateConfig::default()) } diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index ff9cd24f63..1a3e161d5d 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -4507,6 +4507,7 @@ pub async fn run_wait_result_script_by_path_internal( let mut tx = user_db.clone().begin(&authed).await?; let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = script_path_to_payload(script_path, &mut *tx, &w_id, run_query.skip_preprocessor).await?; + drop(tx); let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag, &authed).await?; @@ -4727,6 +4728,7 @@ pub async fn run_wait_result_flow_by_path_internal( edited_by, version, } = get_latest_flow_version_info_for_path(&mut *tx, &w_id, &flow_path, true).await?; + drop(tx); let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag, &authed).await?; diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index d7d9498fab..7fe3ce3392 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -120,6 +120,8 @@ mod scripts; mod service_logs; mod settings; mod slack_approvals; +mod approvals; +mod teams_approvals_ee; #[cfg(feature = "smtp")] mod smtp_server_ee; #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] @@ -650,6 +652,10 @@ pub async fn run_server( "/w/:workspace_id/jobs/slack_approval/:job_id", get(slack_approvals::request_slack_approval), ) + .route( + "/w/:workspace_id/jobs/teams_approval/:job_id", + get(teams_approvals_ee::request_teams_approval), + ) .nest("/w/:workspace_id/github_app", { #[cfg(feature = "enterprise")] { diff --git a/backend/windmill-api/src/slack_approvals.rs b/backend/windmill-api/src/slack_approvals.rs index 4f37b648a9..019f83ecb2 100644 --- a/backend/windmill-api/src/slack_approvals.rs +++ b/backend/windmill-api/src/slack_approvals.rs @@ -3,28 +3,21 @@ use axum::{ Extension, }; use hyper::StatusCode; -use serde::{Deserialize, Serialize}; -use serde_json::value::{RawValue, Value}; - -use sqlx::types::Uuid; -use std::{collections::HashMap, str::FromStr}; - -use regex::Regex; use reqwest::Client; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sqlx::types::Uuid; +use std::collections::HashMap; +use windmill_common::error::Error; +use windmill_common::variables::get_secret_value_as_admin; +use crate::approvals::{ + extract_w_id_from_resume_url, handle_resume_action, ApprovalFormDetails, FieldType, + MessageFormat, QueryDefaultArgsJson, QueryDynamicEnumJson, QueryFlowStepId, QueryMessage, + ResumeFormField, ResumeSchema, +}; use crate::db::{ApiAuthed, DB}; -use crate::jobs::{ - cancel_suspended_job, get_resume_urls_internal, resume_suspended_job, QueryApprover, - QueryOrBody, ResumeUrls, -}; - -use windmill_common::{ - cache, - error::{self, Error}, - jobs::JobKind, - scripts::ScriptHash, - variables::get_secret_value_as_admin, -}; +use crate::jobs::{QueryApprover, ResumeUrls}; #[derive(Deserialize, Debug)] pub struct SlackFormData { @@ -91,53 +84,6 @@ struct SelectedOption { value: String, } -#[derive(Debug, Deserialize, Serialize)] -struct ResumeSchema { - schema: Schema, -} - -#[derive(Debug, Deserialize)] -struct ResumeFormRow { - resume_form: Option, - hide_cancel: Option, -} - -#[derive(Debug, Deserialize, Serialize)] -struct Schema { - order: Vec, - required: Vec, - properties: HashMap, -} - -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "lowercase")] -enum FieldType { - Boolean, - String, - Number, - Integer, - #[serde(other)] - Unknown, -} - -#[derive(Debug, Deserialize, Serialize)] -struct ResumeFormField { - r#type: FieldType, - format: Option, - default: Option, - description: Option, - title: Option, - r#enum: Option>, - #[serde(rename = "enumLabels")] - enum_labels: Option>, - nullable: Option, -} - -#[derive(Deserialize)] -pub struct QueryMessage { - message: Option, -} - #[derive(Deserialize)] pub struct QueryResourcePath { slack_resource_path: String, @@ -148,21 +94,6 @@ pub struct QueryChannelId { channel_id: String, } -#[derive(Deserialize)] -pub struct QueryFlowStepId { - flow_step_id: String, -} - -#[derive(Deserialize, Debug)] -pub struct QueryDefaultArgsJson { - default_args_json: Option, -} - -#[derive(Deserialize, Debug)] -pub struct QueryDynamicEnumJson { - dynamic_enums_json: Option, -} - #[derive(Deserialize, Debug)] struct ModalActionValue { w_id: String, @@ -362,73 +293,21 @@ async fn handle_submission( return Ok(()); } - // Use regex to extract information from private_metadata - let re = Regex::new(r"/api/w/(?P[^/]+)/jobs_u/(?Presume|cancel)/(?P[^/]+)/(?P[^/]+)/(?P[a-fA-F0-9]+)(?:\?approver=(?P[^&]+))?").unwrap(); - let captures = re.captures(resume_url.as_str()).ok_or_else(|| { - tracing::error!("Resume URL does not match the pattern."); - Error::BadRequest("Invalid URL format.".to_string()) - })?; + // Use the common handler to process the resume/cancel action + handle_resume_action(authed, db.clone(), &resume_url, state_json, action).await?; - let (w_id, job_id, resume_id, secret, approver) = ( - captures.name("w_id").map_or("", |m| m.as_str()), - captures.name("job_id").map_or("", |m| m.as_str()), - captures.name("resume_id").map_or("", |m| m.as_str()), - captures.name("secret").map_or("", |m| m.as_str()), - captures.name("approver").map(|m| m.as_str().to_string()), - ); - - let approver = QueryApprover { approver: approver }; - - // Convert job_id and resume_id to appropriate types - let job_uuid = Uuid::from_str(job_id) - .map_err(|_| Error::BadRequest("Invalid job ID format.".to_string()))?; - - let resume_id_parsed = resume_id - .parse::() - .map_err(|_| Error::BadRequest("Invalid resume ID format.".to_string()))?; - - // Call the appropriate function based on the action - let res = if action == "resume" { - resume_suspended_job( - authed, - Extension(db.clone()), - Path(( - w_id.to_string(), - job_uuid, - resume_id_parsed, - secret.to_string(), - )), - Query(approver), - QueryOrBody(Some(state_json)), - ) - .await - } else { - cancel_suspended_job( - authed, - Extension(db.clone()), - Path(( - w_id.to_string(), - job_uuid, - resume_id_parsed, - secret.to_string(), - )), - Query(approver), - QueryOrBody(Some(state_json)), - ) - .await - }; - tracing::debug!("Resume job action result: {:#?}", res); - let slack_token = get_slack_token(&db, &resource_path, &w_id).await?; + let w_id = extract_w_id_from_resume_url(&resume_url)?; + let slack_token = get_slack_token(&db, &resource_path, w_id).await?; update_original_slack_message(action, slack_token, container).await?; Ok(()) } async fn transform_schemas( text: &str, - properties: Option<&HashMap>, + properties: Option>, urls: &ResumeUrls, - order: Option<&Vec>, - required: Option<&Vec>, + order: Option>, + required: Option>, default_args_json: Option<&serde_json::Value>, dynamic_enums_json: Option<&serde_json::Value>, ) -> Result { @@ -443,16 +322,16 @@ async fn transform_schemas( })]; if let Some(properties) = properties { - for key in order.unwrap() { - if let Some(schema) = properties.get(key) { - let is_required = required.unwrap().contains(key); + for key in order.unwrap_or_default() { + if let Some(schema) = properties.get(&key) { + let is_required = required.as_ref().map_or(false, |r| r.contains(&key)); - let default_value = default_args_json.and_then(|json| json.get(key).cloned()); + let default_value = default_args_json.and_then(|json| json.get(&key).cloned()); let dynamic_enums_value = - dynamic_enums_json.and_then(|json| json.get(key).cloned()); + dynamic_enums_json.and_then(|json| json.get(&key).cloned()); let input_block = create_input_block( - key, + &key, schema, is_required, default_value, @@ -947,155 +826,59 @@ async fn get_modal_blocks( default_args_json: Option<&serde_json::Value>, dynamic_enums_json: Option<&serde_json::Value>, ) -> Result, Error> { - let res = get_resume_urls_internal( - axum::Extension(db.clone()), - Path((w_id.to_string(), job_id, resume_id)), - Query(QueryApprover { approver: approver.map(|a| a.to_string()) }), + let approval_details = crate::approvals::get_approval_form_details( + db, + w_id, + job_id, + flow_step_id, + resume_id, + approver, + message, + MessageFormat::Slack, ) .await?; - let urls = res.0; + let ApprovalFormDetails { message_str, urls, schema } = approval_details; - tracing::debug!("Job ID: {:?}", job_id); - - let (job_kind, script_hash, raw_flow, parent_job_id, created_at, created_by, script_path, args) = sqlx::query!( - "SELECT - v2_as_queue.job_kind AS \"job_kind!: JobKind\", - v2_as_queue.script_hash AS \"script_hash: ScriptHash\", - v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json>\", - v2_as_completed_job.parent_job AS \"parent_job: Uuid\", - v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\", - v2_as_completed_job.created_by AS \"created_by!\", - v2_as_queue.script_path, - v2_as_queue.args AS \"args: sqlx::types::Json>\" - FROM v2_as_queue - JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id - WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 - LIMIT 1", - job_id, - &w_id + // Get the card content + let card_content = transform_schemas( + &message_str, + schema + .as_ref() + .and_then(|s| s.resume_form.as_ref()) + .map(|f| { + let inner_schema: ResumeSchema = serde_json::from_value(f.clone()).unwrap(); + inner_schema.schema.properties + }), + &urls, + schema + .as_ref() + .and_then(|s| s.resume_form.as_ref()) + .map(|f| { + let inner_schema: ResumeSchema = serde_json::from_value(f.clone()).unwrap(); + inner_schema.schema.order + }), + schema + .as_ref() + .and_then(|s| s.resume_form.as_ref()) + .map(|f| { + let inner_schema: ResumeSchema = serde_json::from_value(f.clone()).unwrap(); + inner_schema.schema.required + }), + default_args_json, + dynamic_enums_json, ) - .fetch_optional(&db) - .await - .map_err(|e| error::Error::BadRequest(e.to_string()))? - .ok_or_else(|| error::Error::BadRequest("This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string())) - .map(|r| (r.job_kind, r.script_hash, r.raw_flow, r.parent_job, r.created_at, r.created_by, r.script_path, r.args))?; + .await?; - let flow_data = match cache::job::fetch_flow(&db, job_kind, script_hash).await { - Ok(data) => data, - Err(_) => { - if let Some(parent_job_id) = parent_job_id.as_ref() { - cache::job::fetch_preview_flow(&db, parent_job_id, raw_flow).await? - } else { - return Err(error::Error::BadRequest( - "This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string(), - )); - } - } - }; - - let flow_value = &flow_data.flow; - let flow_step_id = flow_step_id.unwrap_or(""); - let module = flow_value.modules.iter().find(|m| m.id == flow_step_id); - - tracing::debug!("Module: {:#?}", module); - - let schema = module.and_then(|module| { - module.suspend.as_ref().map(|suspend| ResumeFormRow { - resume_form: suspend.resume_form.clone(), - hide_cancel: suspend.hide_cancel, - }) - }); - - let args_str = args.map_or("None".to_string(), |a| a.get().to_string()); - let parent_job_id_str = parent_job_id.map_or("None".to_string(), |id| id.to_string()); - let script_path_str = script_path.as_deref().unwrap_or("None"); - - let created_at_formatted = created_at.format("%Y-%m-%d %H:%M:%S").to_string(); - - let mut message_str = format!( - "A workflow has been suspended and is waiting for approval:\n\n\ - *Created by*: {created_by}\n\ - *Created at*: {created_at_formatted}\n\ - *Script path*: {script_path_str}\n\ - *Args*: {args_str}\n\ - *Flow ID*: {parent_job_id_str}\n\n" - ); - - // Append custom message if provided - if let Some(msg) = message { - message_str.push_str(msg); - } - - tracing::debug!("Schema: {:#?}", schema); - - if let Some(resume_schema) = schema { - let hide_cancel = resume_schema.hide_cancel.unwrap_or(false); - - // if hide cancel is false add note to message - if !hide_cancel { - message_str.push_str("\n\n*NOTE*: closing this modal will cancel the workflow.\n\n"); - } - - // Convert message_str back to &str when needed - let message_str_ref: &str = &message_str; - - if let Some(schema_obj) = resume_schema.resume_form { - let inner_schema: ResumeSchema = - serde_json::from_value(schema_obj.clone()).map_err(|e| { - tracing::error!("Failed to deserialize form schema: {:?}", e); - Error::BadRequest( - "Failed to deserialize resume form schema! Unsupported form field used." - .to_string(), - ) - })?; - - let blocks = transform_schemas( - message_str_ref, - Some(&inner_schema.schema.properties), - &urls, - Some(&inner_schema.schema.order), - Some(&inner_schema.schema.required), - default_args_json, - dynamic_enums_json, - ) - .await?; - - tracing::debug!("Slack Blocks: {:#?}", blocks); - return Ok(axum::Json(construct_payload( - blocks, - hide_cancel, - trigger_id, - &urls.resume, - resource_path, - container, - ))); - } else { - tracing::debug!("No suspend form found!"); - let blocks = transform_schemas( - message_str_ref, - None, - &urls, - None, - None, - default_args_json, - dynamic_enums_json, - ) - .await?; - return Ok(axum::Json(construct_payload( - blocks, - hide_cancel, - trigger_id, - &urls.resume, - resource_path, - container, - ))); - } - } else { - Err(Error::BadRequest( - "No approval form schema found.".to_string(), - )) - } + tracing::debug!("Slack Blocks: {:#?}", card_content); + Ok(axum::Json(construct_payload( + card_content, + schema.as_ref().and_then(|s| s.hide_cancel).unwrap_or(false), + trigger_id, + &urls.resume, + resource_path, + container, + ))) } fn construct_payload( diff --git a/backend/windmill-api/src/teams_approvals_ee.rs b/backend/windmill-api/src/teams_approvals_ee.rs new file mode 100644 index 0000000000..05698b98e3 --- /dev/null +++ b/backend/windmill-api/src/teams_approvals_ee.rs @@ -0,0 +1,7 @@ +use hyper::StatusCode; + +use windmill_common::error::Error; + +pub async fn request_teams_approval() -> Result { + Err(Error::InternalErr("enterprise feature only".to_string())) +} \ No newline at end of file diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 6b34e7fa85..75c9a8d034 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -443,10 +443,12 @@ async fn get_settings( "SELECT workspace_id, slack_team_id, teams_team_id, teams_team_name, slack_name, slack_command_script, teams_command_script, slack_email, auto_invite_domain, auto_invite_operator, auto_add, customer_id, plan, webhook, deploy_to, ai_config, error_handler, error_handler_extra_args, error_handler_muted_on_cancel, large_file_storage, git_sync, deploy_ui, default_app, default_scripts, mute_critical_alerts, color, operator_settings, git_app_installations FROM workspace_settings WHERE workspace_id = $1", &w_id ) - .fetch_one(&mut *tx) + .fetch_optional(&mut *tx) .await .map_err(|e| Error::internal_err(format!("getting settings: {e:#}")))?; tx.commit().await?; + let settings = not_found_if_none(settings, "workspace settings", &w_id)?; + Ok(Json(settings)) } diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 981b71eccf..3f1a1bd385 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -719,7 +719,10 @@ impl Future for WarnAfterFuture { // Poll the timeout future to check if it has elapsed. if !*this.warned { if this.timeout.poll(cx).is_ready() { - tracing::warn!(location = this.location, "SLOW_QUERY: query to db taking longer than expected (> {} seconds). This is a sign the database is under heavy load, query is too heavy or database is undersized", + tracing::warn!( + location = this.location, + "SLOW_QUERY: query {} to db taking longer than expected (> {} seconds)", + this.location, this.seconds, ); *this.warned = true; @@ -733,7 +736,8 @@ impl Future for WarnAfterFuture { let elapsed = this.start_time.elapsed(); tracing::warn!( location = this.location, - "SLOW_QUERY: completed with total duration: {:.2?}", + "SLOW_QUERY: completed query {} with total duration: {:.2?}", + this.location, elapsed ); } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 74e002f545..014235eb4a 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -4191,6 +4191,7 @@ pub async fn push<'c, 'd>( job_id, ) .execute(&mut *tx) + .warn_after_seconds(3) .await .map_err(|e| Error::internal_err(format!("Could not insert concurrency_key={concurrency_key} for job_id={job_id} script_path={script_path:?} workspace_id={workspace_id}: {e:#}")))?; } @@ -4356,6 +4357,7 @@ pub async fn push<'c, 'd>( Json(flow_status) as Json, ) .execute(&mut *tx) + .warn_after_seconds(1) .await?; } @@ -4424,6 +4426,7 @@ pub async fn push<'c, 'd>( script_path.as_ref().map(|x| x.as_str()), Some(hm), ) + .warn_after_seconds(1) .await?; } diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 63295208a2..c37e78d7cf 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -57,6 +57,7 @@ sqlx.workspace = true uuid.workspace = true tracing.workspace = true tokio.workspace = true +tokio-stream.workspace = true serde.workspace = true serde_json.workspace = true futures.workspace = true diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 1e47edc67e..7d46fc6040 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -786,14 +786,30 @@ pub async fn handle_ansible_job( if let Ok(lockfile) = serde_json::from_str(s) { Some(lockfile) } else { - append_logs( - &job.id, - &job.workspace_id, - format!("WARN: lockfile could not be parsed: `{s}`"), - conn, - ) - .await; - None + if !s.trim_start().starts_with('{') { + append_logs( + &job.id, + &job.workspace_id, + format!("WARN: lockfile seems to be in an older version, roles and collections are therefore using the latest version and not the one locked at deployment. Redeploy the script to correct this"), + conn, + ) + .await; + Some(AnsibleDependencyLocks { + python_lockfile: s.to_string(), + git_repos: HashMap::new(), + collections_and_roles: String::new(), + collections_and_roles_logs: String::new(), + }) + } else { + append_logs( + &job.id, + &job.workspace_id, + format!("WARN: lockfile could not be parsed: {s}"), + conn, + ) + .await; + None + } } } else { None @@ -948,7 +964,13 @@ pub async fn handle_ansible_job( let empty = String::new(); let (lockfile, logs) = req_lockfiles .as_ref() - .map(|r| (&r.collections_and_roles, &r.collections_and_roles_logs)) + .and_then(|r| { + if r.collections_and_roles.is_empty() { + None + } else { + Some((&r.collections_and_roles, &r.collections_and_roles_logs)) + } + }) .unwrap_or((collections, &empty)); if !logs.is_empty() { diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 7a7a78282c..9683c0d8d5 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -285,6 +285,8 @@ async fn handle_docker_job( occupancy_metrics: &mut OccupancyMetrics, killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, ) -> Result, Error> { + use crate::job_logger::append_logs_with_compaction; + let client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow)?; let container_id = job_id.to_string(); @@ -313,6 +315,7 @@ async fn handle_docker_job( let w_id = workspace_id.to_string(); let j_id = job_id.clone(); let conn2 = conn.clone(); + let worker_name2 = worker_name.to_string(); let (tx, mut rx) = tokio::sync::broadcast::channel::<()>(1); let mut killpill_rx = killpill_rx.resubscribe(); @@ -334,7 +337,21 @@ async fn handle_docker_job( log = log_stream.next() => { match log { Some(Ok(log)) => { - append_logs(&j_id, w_id.clone(), log.to_string(), &conn2).await; + match &conn2 { + Connection::Sql(db) => { + append_logs_with_compaction( + &j_id, + &w_id, + &log.to_string(), + &db, + &worker_name2, + ) + .await; + } + c @ Connection::Http(_) => { + append_logs(&j_id, &w_id, &log.to_string(), &c).await; + } + } } Some(Err(e)) => { tracing::error!("Error getting logs: {:?}", e); diff --git a/backend/windmill-worker/src/dedicated_worker.rs b/backend/windmill-worker/src/dedicated_worker.rs index 33a83a1492..9eb3c09da3 100644 --- a/backend/windmill-worker/src/dedicated_worker.rs +++ b/backend/windmill-worker/src/dedicated_worker.rs @@ -187,14 +187,14 @@ pub async fn handle_dedicated_process( let result = Arc::new(result); append_logs(&job.id, &job.workspace_id, logs.clone(), &db.into()).await; if line.starts_with("wm_res[success]:") { - job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap() + job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None }, true).await.unwrap() } else { - job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap() + job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }, true).await.unwrap() } }, Err(e) => { tracing::error!("Could not deserialize job result `{line}`: {e:?}"); - job_completed_tx.send_job(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap(); + job_completed_tx.send_job(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }, true).await.unwrap(); }, }; logs = init_log.clone(); diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 6cf4272a58..d82a72df98 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -130,13 +130,13 @@ pub async fn handle_child( } else { tracing::info!("could not get child pid"); } - let (set_too_many_logs, mut too_many_logs) = watch::channel::(false); + let (mut set_too_many_logs, mut too_many_logs) = watch::channel::(false); let (tx, rx) = broadcast::channel::<()>(3); - let mut rx2 = tx.subscribe(); + let mut rx2: broadcast::Receiver<()> = tx.subscribe(); let output = child_joined_output_stream(&mut child, job_id.clone()); - let job_id = job_id.clone(); + let job_id: Uuid = job_id.clone(); /* the cancellation future is polled on by `wait_on_child` while * waiting for the child to exit normally */ @@ -296,147 +296,19 @@ pub async fn handle_child( }; /* a future that reads output from the child and appends to the database */ - let lines = async move { - - let max_log_size = if *CLOUD_HOSTED { - MAX_RESULT_SIZE - } else { - usize::MAX - }; - - /* log_remaining is zero when output limit was reached */ - let mut log_remaining = if *CLOUD_HOSTED { - max_log_size - } else { - usize::MAX - }; - let mut result = io::Result::Ok(()); - let mut output = output.take_until(async { - let _ = rx2.recv().await; - //wait at most 50ms after end of a script for output stream to end - tokio::time::sleep(Duration::from_millis(50)).await; - }).boxed(); - /* `do_write` resolves the task, but does not contain the Result. - * It's useful to know if the task completed. */ - let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle(); - - let mut log_total_size: u64 = 0; - let pg_log_total_size = Arc::new(AtomicU32::new(0)); - - let mut pipe_stdout = pipe_stdout; - - while let Some(line) = output.by_ref().next().await { - - let do_write_ = do_write.shared(); - - let delay = if start.elapsed() < Duration::from_secs(10) { - Duration::from_millis(500) - } else if start.elapsed() < Duration::from_secs(60){ - Duration::from_millis(2500) - } else { - Duration::from_millis(5000) - }; - - let delay = if *SLOW_LOGS { - delay * 10 - } else { - delay - }; - - let mut read_lines = stream::once(async { line }) - .chain(output.by_ref()) - /* after receiving a line, continue until some delay has passed - * _and_ the previous database write is complete */ - .take_until(future::join(sleep(delay), do_write_.clone())) - .boxed(); - - /* Read up until an error is encountered, - * handle log lines first and then the error... */ - let mut joined = String::new(); - - while let Some(line) = read_lines.next().await { - - match line { - Ok(line) => { - if line.is_empty() { - continue; - } - append_with_limit(&mut joined, &line, &mut log_remaining); - if log_remaining == 0 { - tracing::info!(%job_id, "Too many logs lines for job {job_id}"); - let _ = set_too_many_logs.send(true); - joined.push_str(&format!( - "Job logs or result reached character limit of {MAX_RESULT_SIZE}; killing job." - )); - /* stop reading and drop our streams fairly quickly */ - break; - } - } - Err(err) => { - result = Err(err); - break; - } - } - } - - - /* Ensure the last flush completed before starting a new one. - * - * This shouldn't pause since `take_until()` reads lines until `do_write` - * resolves. We only stop reading lines before `take_until()` resolves if we reach - * EOF or a read error. In those cases, waiting on a database query to complete is - * fine because we're done. */ - - if let Some(Ok(p)) = do_write_ - .then(|()| write_result) - .await - .err() - .map(|err| err.try_into_panic()) - { - panic::resume_unwind(p); - } - - - let joined_len = joined.len() as u64; - log_total_size += joined_len; - let compact_logs = log_total_size > LARGE_LOG_THRESHOLD_SIZE as u64; - if compact_logs { - log_total_size = 0; - } - - let worker_name = worker.to_string(); - let w_id2 = w_id.to_string(); - - - if let Some(buf) = &mut pipe_stdout { - buf.push_str(&joined); - (do_write, write_result) = tokio::spawn(async { }).remote_handle(); - } else { - (do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, conn.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle(); - } - - if let Err(err) = result { - tracing::error!(%job_id, %err, "error reading output for job {job_id} '{child_name}': {err}"); - break; - } - - if *set_too_many_logs.borrow() { - break; - } - } - - /* drop our end of the pipe */ - drop(output); - - if let Some(Ok(p)) = do_write - .then(|()| write_result) - .await - .err() - .map(|err| err.try_into_panic()) - { - panic::resume_unwind(p); - } - }.instrument(trace_span!("child_lines")); + let lines = write_lines( + output, + &job_id, + w_id, + worker, + conn, + &mut set_too_many_logs, + start, + pipe_stdout, + &mut rx2, + child_name, + ) + .instrument(trace_span!("child_lines")); let (wait_result, _) = tokio::join!(wait_on_child, lines); @@ -462,6 +334,169 @@ pub async fn handle_child( } } +pub async fn write_lines( + output: impl stream::Stream> + Send, + job_id: &Uuid, + w_id: &str, + worker: &str, + conn: &Connection, + set_too_many_logs: &mut watch::Sender, + start: Instant, + pipe_stdout: Option<&mut String>, + rx2: &mut broadcast::Receiver<()>, + child_name: &str, +) { + let max_log_size = if *CLOUD_HOSTED { + MAX_RESULT_SIZE + } else { + usize::MAX + }; + + /* log_remaining is zero when output limit was reached */ + let mut log_remaining = if *CLOUD_HOSTED { + max_log_size + } else { + usize::MAX + }; + let mut result = io::Result::Ok(()); + let mut output = output + .take_until(async { + let _ = rx2.recv().await; + //wait at most 50ms after end of a script for output stream to end + tokio::time::sleep(Duration::from_millis(50)).await; + }) + .boxed(); + /* `do_write` resolves the task, but does not contain the Result. + * It's useful to know if the task completed. */ + let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle(); + + let mut log_total_size: u64 = 0; + let pg_log_total_size = Arc::new(AtomicU32::new(0)); + + let mut pipe_stdout = pipe_stdout; + + while let Some(line) = output.by_ref().next().await { + let do_write_ = do_write.shared(); + + let delay = if start.elapsed() < Duration::from_secs(10) { + Duration::from_millis(500) + } else if start.elapsed() < Duration::from_secs(60) { + Duration::from_millis(2500) + } else { + Duration::from_millis(5000) + }; + + let delay = if *SLOW_LOGS { delay * 10 } else { delay }; + + let mut read_lines = stream::once(async { line }) + .chain(output.by_ref()) + /* after receiving a line, continue until some delay has passed + * _and_ the previous database write is complete */ + .take_until(future::join(sleep(delay), do_write_.clone())) + .boxed(); + + /* Read up until an error is encountered, + * handle log lines first and then the error... */ + let mut joined = String::new(); + + let job_id = job_id.clone(); + while let Some(line) = read_lines.next().await { + match line { + Ok(line) => { + if line.is_empty() { + continue; + } + append_with_limit(&mut joined, &line, &mut log_remaining); + if log_remaining == 0 { + tracing::info!(%job_id, "Too many logs lines for job {job_id}"); + let _ = set_too_many_logs.send(true); + joined.push_str(&format!( + "Job logs or result reached character limit of {MAX_RESULT_SIZE}; killing job." + )); + /* stop reading and drop our streams fairly quickly */ + break; + } + } + Err(err) => { + result = Err(err); + break; + } + } + } + + /* Ensure the last flush completed before starting a new one. + * + * This shouldn't pause since `take_until()` reads lines until `do_write` + * resolves. We only stop reading lines before `take_until()` resolves if we reach + * EOF or a read error. In those cases, waiting on a database query to complete is + * fine because we're done. */ + + if let Some(Ok(p)) = do_write_ + .then(|()| write_result) + .await + .err() + .map(|err| err.try_into_panic()) + { + panic::resume_unwind(p); + } + + let joined_len = joined.len() as u64; + log_total_size += joined_len; + let compact_logs = log_total_size > LARGE_LOG_THRESHOLD_SIZE as u64; + if compact_logs { + log_total_size = 0; + } + + let worker_name = worker.to_string(); + + if let Some(buf) = &mut pipe_stdout { + buf.push_str(&joined); + (do_write, write_result) = tokio::spawn(async {}).remote_handle(); + } else { + let conn = conn.clone(); + let worker_name = worker_name.to_string(); + let w_id = w_id.to_string(); + let job_id = job_id.clone(); + let pg_log_total_size = pg_log_total_size.clone(); + + (do_write, write_result) = tokio::spawn(async move { + append_job_logs( + &job_id, + &w_id, + &joined, + &conn, + compact_logs, + pg_log_total_size, + &worker_name, + ) + .await; + }) + .remote_handle(); + } + + if let Err(err) = result { + tracing::error!(%job_id, %err, "error reading output for job {job_id} '{child_name}': {err}"); + break; + } + + if *set_too_many_logs.borrow() { + break; + } + } + + /* drop our end of the pipe */ + drop(output); + + if let Some(Ok(p)) = do_write + .then(|()| write_result) + .await + .err() + .map(|err| err.try_into_panic()) + { + panic::resume_unwind(p); + } +} + pub(crate) async fn get_mem_peak(pid: Option, nsjail: bool) -> i32 { if pid.is_none() { return -1; diff --git a/backend/windmill-worker/src/job_logger.rs b/backend/windmill-worker/src/job_logger.rs index 97784340c5..f4abd1d0c2 100644 --- a/backend/windmill-worker/src/job_logger.rs +++ b/backend/windmill-worker/src/job_logger.rs @@ -1,8 +1,10 @@ use regex::Regex; pub use windmill_common::jobs::LARGE_LOG_THRESHOLD_SIZE; +use windmill_common::utils::WarnAfterExt; use windmill_common::worker::{Connection, CLOUD_HOSTED}; +use windmill_common::DB; use windmill_queue::append_logs; use std::sync::atomic::AtomicU32; @@ -26,23 +28,23 @@ pub enum CompactLogs { } pub async fn append_job_logs( - job_id: Uuid, - w_id: String, - logs: String, - conn: Connection, + job_id: &Uuid, + w_id: &str, + logs: &str, + conn: &Connection, must_compact_logs: bool, total_size: Arc, - worker_name: String, + worker_name: &str, ) -> () { match conn { Connection::Sql(db) if must_compact_logs => { #[cfg(all(feature = "enterprise", feature = "parquet"))] - s3_storage(job_id, &w_id, &db, logs, total_size, &worker_name).await; + s3_storage(&job_id, &w_id, &db, logs, total_size, worker_name).await; #[cfg(not(all(feature = "enterprise", feature = "parquet")))] { default_disk_log_storage( - job_id, + &job_id, &w_id, &db, logs, @@ -59,6 +61,45 @@ pub async fn append_job_logs( } } +pub async fn append_logs_with_compaction( + job_id: &Uuid, + w_id: &str, + logs: &str, + db: &DB, + worker_name: &str, +) { + let log_length = sqlx::query_scalar!( + "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, $1::text) RETURNING length(logs)", + logs, + job_id, + &w_id, + ) + .fetch_one(db) + .warn_after_seconds(1) + .await; + match log_length { + Ok(length) => { + let len = length.unwrap_or(0); + let conn: Connection = db.into(); + if len > LARGE_LOG_THRESHOLD_SIZE as i32 { + append_job_logs( + &job_id, + w_id, + "", + &conn, + true, + Arc::new(AtomicU32::new(len as u32)), + worker_name, + ) + .await; + } + } + Err(err) => { + tracing::error!(%job_id, %err, "error updating logs for job {job_id}: {err}"); + } + } +} + lazy_static::lazy_static! { static ref RE_00: Regex = Regex::new('\u{00}'.to_string().as_str()).unwrap(); pub static ref NO_LOGS_AT_ALL: bool = std::env::var("NO_LOGS_AT_ALL").ok().is_some_and(|x| x == "1" || x == "true"); diff --git a/backend/windmill-worker/src/job_logger_ee.rs b/backend/windmill-worker/src/job_logger_ee.rs index 43cb43f673..4b1d34392c 100644 --- a/backend/windmill-worker/src/job_logger_ee.rs +++ b/backend/windmill-worker/src/job_logger_ee.rs @@ -9,22 +9,22 @@ use crate::job_logger::CompactLogs; #[cfg(all(feature = "enterprise", feature = "parquet"))] pub(crate) async fn s3_storage( - _job_id: Uuid, - _w_id: &String, + _job_id: &Uuid, + _w_id: &str, _db: &sqlx::Pool, - _logs: String, + _logs: &str, _total_size: Arc, - _worker_name: &String, + _worker_name: &str, ) { tracing::info!("Logs length of {_job_id} has exceeded a threshold. Implementation to store excess on s3 in not OSS"); } #[allow(dead_code)] pub(crate) async fn default_disk_log_storage( - job_id: Uuid, + job_id: &Uuid, _w_id: &str, _db: &DB, - _nlogs: String, + _logs: &str, _total_size: Arc, _compact_kind: CompactLogs, _worker_name: &str, diff --git a/backend/windmill-worker/src/oracledb_executor.rs b/backend/windmill-worker/src/oracledb_executor.rs index 50ea9a2ec3..3286ad0333 100644 --- a/backend/windmill-worker/src/oracledb_executor.rs +++ b/backend/windmill-worker/src/oracledb_executor.rs @@ -1,26 +1,31 @@ use anyhow::anyhow; use chrono::Utc; -use std::{collections::HashMap, str::FromStr, sync::Arc}; +use std::{collections::HashMap, str::FromStr, sync::Arc, vec}; use windmill_parser::Arg; -use futures::{future::BoxFuture, FutureExt}; +use futures::{future::BoxFuture, FutureExt, StreamExt}; use itertools::Itertools; use oracle::sql_type::{InnerValue, OracleType, ToSql}; use serde::{Deserialize, Serialize}; use serde_json::{json, value::RawValue, Value}; use windmill_common::{ error::{to_anyhow, Error}, + s3_helpers::convert_json_line_stream, worker::{to_raw_value, Connection}, }; use windmill_queue::MiniPulledJob; use windmill_parser_sql::{ - parse_db_resource, parse_oracledb_sig, parse_sql_blocks, parse_sql_statement_named_params, + parse_db_resource, parse_oracledb_sig, parse_s3_mode, parse_sql_blocks, + parse_sql_statement_named_params, }; use windmill_queue::CanceledBy; use crate::{ - common::{build_args_values, check_executor_binary_exists, OccupancyMetrics}, + common::{ + build_args_values, check_executor_binary_exists, s3_mode_args_to_worker_data, + OccupancyMetrics, S3ModeWorkerData, + }, handle_child::run_future_with_polling_update_job_poller, sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args, AuthedClient, @@ -43,6 +48,7 @@ pub fn do_oracledb_inner<'a>( conn: Arc>, column_order: Option<&'a mut Option>>, skip_collect: bool, + s3: Option, ) -> windmill_common::error::Result>>> { let qw = query.trim_end_matches(';').to_string(); @@ -81,55 +87,90 @@ pub fn do_oracledb_inner<'a>( Ok(to_raw_value(&Value::Array(vec![]))) } else { - let rows = tokio::task::spawn_blocking(move || { - let params2: Vec<(&str, &dyn ToSql)> = params - .iter() - .filter(|(k, _)| param_names.contains(&k.clone().into_bytes())) - .map(|(key, val)| (key.as_str(), &**val as &dyn ToSql)) - .collect(); + // We use an mpsc because we need an async stream for s3 mode. However since everything is sync + // in rust-oracle, I assumed that calling ResultSet::next() is blocking when it has to refetch. + let (tx, rx) = tokio::sync::mpsc::channel::>(1000); + let (column_order_oneshot_tx, column_order_oneshot_rx) = + tokio::sync::oneshot::channel::>>(); + let mut column_order_oneshot_tx = Some(column_order_oneshot_tx); + let rows_stream = tokio_stream::wrappers::ReceiverStream::new(rx); + tokio::task::spawn_blocking(move || { + let result = (|| { + let tx = tx.clone(); + let params2: Vec<(&str, &dyn ToSql)> = params + .iter() + .filter(|(k, _)| param_names.contains(&k.clone().into_bytes())) + .map(|(key, val)| (key.as_str(), &**val as &dyn ToSql)) + .collect(); - let c = conn.lock()?; - let mut stmt = c.statement(&qw).build()?; + let c = conn.lock()?; + let mut stmt = c.statement(&qw).build()?; - let rows = match stmt.statement_type() { - oracle::StatementType::Select => { - let result_rows = stmt.query_named(¶ms2)?; - let rows: Vec = - result_rows.into_iter().filter_map(Result::ok).collect_vec(); - rows - } - _ => { - stmt.execute_named(¶ms2)?; - c.commit()?; - vec![] - } - }; + match stmt.statement_type() { + oracle::StatementType::Select => { + let mut result_rows = stmt.query_named(¶ms2)?.enumerate(); + while let Some((i, row)) = result_rows.next() { + match row { + Ok(row) => { + // If first row, infer column order and send it to the channel + if i == 0 { + let col_order: Vec = row + .column_info() + .iter() + .map(|x| x.name().to_string()) + .collect::>(); + let _ = column_order_oneshot_tx + .take() + .unwrap() + .send(Some(col_order)); + } - oracle::Result::Ok(rows) - }) - .await - .map_err(to_anyhow)? - .map_err(to_anyhow)?; + // called in a spawn_blocking synchronous context, unwrap won't panic + tx.blocking_send(Ok(convert_row_to_value(row))).unwrap() + } + Err(e) => { + tx.blocking_send(Err(e)).unwrap(); + break; + } + } + } + } + _ => { + stmt.execute_named(¶ms2)?; + c.commit()?; + } + }; + drop(column_order_oneshot_tx); + Ok::<_, oracle::Error>(()) + })(); + match result { + Ok(_) => {} + Err(e) => tx.blocking_send(Err(e)).unwrap(), + } + // all instances of tx should be dropped here + }); - if let Some(column_order) = column_order { - *column_order = Some( - rows.first() - .map(|x| { - x.column_info() - .iter() - .map(|x| x.name().to_string()) - .collect::>() - }) - .unwrap_or_default(), - ); + if let Ok(Some(col_order)) = column_order_oneshot_rx.await { + if let Some(column_order) = column_order { + *column_order = Some(col_order); + } } - Ok(to_raw_value( - &rows - .into_iter() - .map(|x| convert_row_to_value(x)) - .collect::>(), - )) + if let Some(s3) = s3 { + let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?; + s3.upload(stream.boxed()).await?; + return Ok(serde_json::value::to_raw_value(&s3.object_key)?); + } else { + let rows: Vec<_> = rows_stream.collect().await; + Ok(to_raw_value( + &rows + .into_iter() + .collect::, _>>() + .map_err(to_anyhow)? + .into_iter() + .collect::>(), + )) + } } }; @@ -312,6 +353,7 @@ pub async fn do_oracledb( let job_args = build_args_values(job, client, conn).await?; let inline_db_res_path = parse_db_resource(&query); + let s3 = parse_s3_mode(&query)?.map(|s3| s3_mode_args_to_worker_data(s3, client.clone(), job)); let db_arg = if let Some(inline_db_res_path) = inline_db_res_path { Some( @@ -376,6 +418,7 @@ pub async fn do_oracledb( conn_a.clone(), None, annotations.return_last_result && i < queries.len() - 1, + s3.clone(), )? .await?; res.push(r); @@ -390,7 +433,14 @@ pub async fn do_oracledb( f.boxed() } else { - do_oracledb_inner(&query, statement_values, conn_a, Some(column_order), false)? + do_oracledb_inner( + &query, + statement_values, + conn_a, + Some(column_order), + false, + s3, + )? }; let result = run_future_with_polling_update_job_poller( diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 0bde6fcdc6..7df4521901 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -34,7 +34,7 @@ use windmill_queue::{ use serde_json::{json, value::RawValue}; -use tokio::{sync::broadcast, task::JoinHandle}; +use tokio::task::JoinHandle; use windmill_queue::{add_completed_job, add_completed_job_error}; @@ -43,7 +43,8 @@ use crate::{ common::{error_to_value, read_result, save_in_cache}, otel_ee::add_root_flow_job_to_otlp, worker_flow::update_flow_status_after_job_completion, - AuthedClient, JobCompletedSender, SameWorkerSender, SendResult, INIT_SCRIPT_TAG, + AuthedClient, JobCompletedReceiver, JobCompletedSender, SameWorkerSender, SendResult, + UpdateFlow, INIT_SCRIPT_TAG, }; async fn process_jc( @@ -118,7 +119,7 @@ async fn process_jc( } pub fn start_background_processor( - job_completed_rx: flume::Receiver, + job_completed_rx: JobCompletedReceiver, job_completed_sender: JobCompletedSender, same_worker_queue_size: Arc, job_completed_processor_is_done: Arc, @@ -127,13 +128,14 @@ pub fn start_background_processor( worker_dir: String, same_worker_tx: SameWorkerSender, worker_name: String, - mut killpill_rx: broadcast::Receiver<()>, killpill_tx: KillpillSender, is_dedicated_worker: bool, ) -> JoinHandle<()> { tokio::spawn(async move { let mut has_been_killed = false; + let JobCompletedReceiver { bounded_rx, mut killpill_rx, unbounded_rx } = job_completed_rx; + #[cfg(feature = "benchmark")] let mut infos = BenchmarkInfo::new(); @@ -144,15 +146,21 @@ pub fn start_background_processor( //if we have been killed, we want to drain the queue of jobs while let Some(sr) = { if has_been_killed && same_worker_queue_size.load(Ordering::SeqCst) == 0 { - job_completed_rx + unbounded_rx .try_recv() .ok() .map(JobCompletedRx::JobCompleted) + .or_else(|| bounded_rx.try_recv().ok().map(JobCompletedRx::JobCompleted)) } else { tokio::select! { - result = job_completed_rx.recv_async() => { + biased; + result = unbounded_rx.recv_async() => { result.ok().map(JobCompletedRx::JobCompleted) } + result = bounded_rx.recv_async() => { + result.ok().map(JobCompletedRx::JobCompleted) + } + _ = killpill_rx.recv() => { Some(JobCompletedRx::Killpill) } @@ -207,7 +215,7 @@ pub fn start_background_processor( infos.add_iter(bench, true); } } - JobCompletedRx::JobCompleted(SendResult::UpdateFlow { + JobCompletedRx::JobCompleted(SendResult::UpdateFlow(UpdateFlow { flow, w_id, success, @@ -215,7 +223,7 @@ pub fn start_background_processor( worker_dir, stop_early_override, token, - }) => { + })) => { // let r; tracing::info!(parent_flow = %flow, "updating flow status"); if let Err(e) = update_flow_status_after_job_completion( @@ -288,7 +296,7 @@ async fn send_job_completed( duration, }; job_completed_tx - .send_job(jc) + .send_job(jc, true) .with_context(windmill_common::otel_ee::otel_ctx()) .await .expect("send job completed") diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 8590603c2f..46a7c5b098 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -568,26 +568,44 @@ pub struct SameWorkerSender(pub Sender, pub Arc); #[allow(dead_code)] #[derive(Clone)] pub enum JobCompletedSender { - Sql(flume::Sender, broadcast::Sender<()>), + Sql(SqlJobCompletedSender), Http(HttpClient), NeverUsed, } +#[derive(Clone)] +pub struct SqlJobCompletedSender { + sender: flume::Sender, + unbounded_sender: flume::Sender, + killpill_tx: broadcast::Sender<()>, +} + +pub struct JobCompletedReceiver { + pub bounded_rx: flume::Receiver, + pub killpill_rx: broadcast::Receiver<()>, + pub unbounded_rx: flume::Receiver, +} + +impl JobCompletedReceiver { + pub fn clone(&self) -> Self { + Self { + bounded_rx: self.bounded_rx.clone(), + killpill_rx: self.killpill_rx.resubscribe(), + unbounded_rx: self.unbounded_rx.clone(), + } + } +} + impl JobCompletedSender { - pub fn new( - conn: &Connection, - buffer_size: usize, - ) -> ( - Self, - Option<(flume::Receiver, broadcast::Receiver<()>)>, - ) { + pub fn new(conn: &Connection, buffer_size: u8) -> (Self, Option) { match conn { Connection::Sql(_) => { - let (sender, receiver) = flume::bounded::(buffer_size); - let (killpill_tx, killpill_rx) = broadcast::channel::<()>(buffer_size); + let (sender, receiver) = flume::bounded::(buffer_size as usize); + let (unbounded_sender, unbounded_rx) = flume::unbounded::(); + let (killpill_tx, killpill_rx) = broadcast::channel::<()>(10); ( - Self::Sql(sender, killpill_tx), - Some((receiver, killpill_rx)), + Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, killpill_tx }), + Some(JobCompletedReceiver { bounded_rx: receiver, killpill_rx, unbounded_rx }), ) } Connection::Http(client) => (Self::Http(client.clone()), None), @@ -597,14 +615,20 @@ impl JobCompletedSender { (Self::NeverUsed, None) } - pub async fn send_job(&self, jc: JobCompleted) -> anyhow::Result<()> { + pub async fn send_job(&self, jc: JobCompleted, wait_for_capacity: bool) -> anyhow::Result<()> { match self { - Self::Sql(sender, _) => sender + Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, .. }) => { + if wait_for_capacity { + sender + } else { + unbounded_sender + } .send_async(SendResult::JobCompleted(jc)) .await .map_err(|_e| { anyhow::anyhow!("Failed to send job completed to background processor") - }), + }) + } Self::Http(client) => { crate::agent_workers::send_result(client, jc).await?; Ok(()) @@ -618,9 +642,19 @@ impl JobCompletedSender { } } - pub async fn send(&self, send_result: SendResult) -> Result<(), flume::SendError> { + pub async fn send( + &self, + send_result: SendResult, + wait_for_capacity: bool, + ) -> Result<(), flume::SendError> { match self { - Self::Sql(sender, _) => sender.send_async(send_result).await, + Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, .. }) => { + if wait_for_capacity { + sender.send_async(send_result).await + } else { + unbounded_sender.send_async(send_result).await + } + } Self::Http(_) => { tracing::error!("Sending job completed to http client, this should not happen"); Ok(()) @@ -636,7 +670,7 @@ impl JobCompletedSender { pub async fn kill(&self) -> Result<(), broadcast::error::SendError<()>> { match self { - Self::Sql(_, killpill_tx) => { + Self::Sql(SqlJobCompletedSender { killpill_tx, .. }) => { tracing::info!("Sending killpill to bg processors"); killpill_tx.send(())?; Ok(()) @@ -1057,7 +1091,7 @@ pub async fn run_worker( let (same_worker_tx, mut same_worker_rx) = mpsc::channel::(5); - let (job_completed_tx, job_completed_rx) = JobCompletedSender::new(&conn, 3); + let (job_completed_tx, job_completed_rx) = JobCompletedSender::new(&conn, 10); let same_worker_queue_size = Arc::new(AtomicU16::new(0)); let same_worker_tx = SameWorkerSender(same_worker_tx, same_worker_queue_size.clone()); @@ -1065,22 +1099,19 @@ pub async fn run_worker( Arc::new(AtomicBool::new(matches!(conn, Connection::Http(_)))); let send_result = match (conn, job_completed_rx) { - (Connection::Sql(db), Some((job_completed_rx, bg_killpill_rx))) => { - Some(start_background_processor( - job_completed_rx, - job_completed_tx.clone(), - same_worker_queue_size.clone(), - job_completed_processor_is_done.clone(), - base_internal_url.to_string(), - db.clone(), - worker_dir.clone(), - same_worker_tx.clone(), - worker_name.clone(), - bg_killpill_rx, - killpill_tx.clone(), - is_dedicated_worker, - )) - } + (Connection::Sql(db), Some(job_completed_receiver)) => Some(start_background_processor( + job_completed_receiver, + job_completed_tx.clone(), + same_worker_queue_size.clone(), + job_completed_processor_is_done.clone(), + base_internal_url.to_string(), + db.clone(), + worker_dir.clone(), + same_worker_tx.clone(), + worker_name.clone(), + killpill_tx.clone(), + is_dedicated_worker, + )), _ => None, }; @@ -1482,17 +1513,20 @@ pub async fn run_worker( if matches!(job.kind, JobKind::Noop) { add_time!(bench, "send job completed START"); job_completed_tx - .send_job(JobCompleted { - job: Arc::new(job.job()), - success: true, - result: Arc::new(empty_result()), - result_columns: None, - mem_peak: 0, - cached_res_path: None, - token: "".to_string(), - canceled_by: None, - duration: None, - }) + .send_job( + JobCompleted { + job: Arc::new(job.job()), + success: true, + result: Arc::new(empty_result()), + result_columns: None, + mem_peak: 0, + cached_res_path: None, + token: "".to_string(), + canceled_by: None, + duration: None, + }, + true, + ) .await .expect("send job completed END"); add_time!(bench, "sent job completed"); @@ -1706,21 +1740,24 @@ pub async fn run_worker( } Connection::Http(_) => { job_completed_tx - .send_job(JobCompleted { - job: arc_job.clone(), - result: Arc::new( - windmill_common::worker::to_raw_value( - &error_to_value(err), + .send_job( + JobCompleted { + job: arc_job.clone(), + result: Arc::new( + windmill_common::worker::to_raw_value( + &error_to_value(err), + ), ), - ), - result_columns: None, - mem_peak: 0, - canceled_by: None, - success: false, - cached_res_path: None, - token: authed_client.token.clone(), - duration: None, - }) + result_columns: None, + mem_peak: 0, + canceled_by: None, + success: false, + cached_res_path: None, + token: authed_client.token.clone(), + duration: None, + }, + false, + ) .await .expect("send job completed"); } @@ -1885,15 +1922,17 @@ async fn queue_init_bash_maybe<'c>( pub enum SendResult { JobCompleted(JobCompleted), - UpdateFlow { - flow: Uuid, - w_id: String, - success: bool, - result: Box, - worker_dir: String, - stop_early_override: Option, - token: String, - }, + UpdateFlow(UpdateFlow), +} + +pub struct UpdateFlow { + pub flow: Uuid, + pub w_id: String, + pub success: bool, + pub result: Box, + pub worker_dir: String, + pub stop_early_override: Option, + pub token: String, } async fn do_nativets( @@ -2065,17 +2104,20 @@ async fn handle_queued_job( append_logs(&job.id, &job.workspace_id, logs, conn).await; } job_completed_tx - .send_job(JobCompleted { - job, - result, - result_columns: None, - mem_peak: 0, - canceled_by: None, - success: true, - cached_res_path: None, - token: client.token.clone(), - duration: None, - }) + .send_job( + JobCompleted { + job, + result, + result_columns: None, + mem_peak: 0, + canceled_by: None, + success: true, + cached_res_path: None, + token: client.token.clone(), + duration: None, + }, + true, + ) .await .expect("send job completed"); diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 3a88d50f9f..53ae99db20 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -14,7 +14,8 @@ use std::time::Duration; use crate::common::{cached_result_path, save_in_cache}; use crate::js_eval::{eval_timeout, IdContext}; use crate::{ - AuthedClient, JobCompletedSender, PreviousResult, SameWorkerSender, SendResult, KEEP_JOB_DIR, + AuthedClient, JobCompletedSender, PreviousResult, SameWorkerSender, SendResult, UpdateFlow, + KEEP_JOB_DIR, }; use anyhow::Context; use futures::TryFutureExt; @@ -1539,22 +1540,47 @@ pub async fn handle_flow( ); } } - let mut rec = Some(PushNextFlowJobRec { flow_job: flow_job, status: status }); - while let Some(nrec) = rec { - rec = push_next_flow_job( - nrec.flow_job, - nrec.status, + let mut rec = PushNextFlowJobRec { flow_job: flow_job, status: status }; + loop { + let PushNextFlowJobRec { flow_job, status } = rec; + let next = push_next_flow_job( + flow_job, + status, flow, db, client, last_result.clone(), same_worker_tx.clone(), worker_dir, - job_completed_tx.clone(), worker_name, ) .warn_after_seconds(10) .await?; + match next { + PushNextFlowJob::Rec(nrec) => { + tracing::info!("recursively pushing next flow job {}", nrec.flow_job.id); + rec = nrec; + } + PushNextFlowJob::Done(update_flow) => { + if let Some(update_flow) = update_flow { + tracing::info!( + "sending flow status update {} with success {} to job completed channel", + update_flow.flow, + update_flow.success + ); + job_completed_tx + .send(SendResult::UpdateFlow(update_flow), false) + .warn_after_seconds(3) + .await + .map_err(|e| { + Error::internal_err(format!( + "error sending update flow message to job completed channel: {e:#}" + )) + })?; + } + break; + } + } } Ok(()) @@ -1600,6 +1626,10 @@ lazy_static::lazy_static! { pub static ref EHM: HashMap> = HashMap::new(); } +enum PushNextFlowJob { + Rec(PushNextFlowJobRec), + Done(Option), +} struct PushNextFlowJobRec { flow_job: Arc, status: FlowStatus, @@ -1615,9 +1645,8 @@ async fn push_next_flow_job( last_job_result: Option>>, same_worker_tx: SameWorkerSender, worker_dir: &str, - job_completed_tx: JobCompletedSender, worker_name: &str, -) -> error::Result> { +) -> error::Result { let job_root = flow_job .flow_innermost_root_job .map(|x| x.to_string()) @@ -1650,29 +1679,20 @@ async fn push_next_flow_job( // if this is an empty module of if the module has already been completed, successfully, update the parent flow if flow.modules.is_empty() || matches!(status_module, FlowStatusModule::Success { .. }) { - job_completed_tx - .send(SendResult::UpdateFlow { - flow: flow_job.id, - success: true, - result: if flow.modules.is_empty() { - to_raw_value(arc_flow_job_args.as_ref()) - } else { - // it has to be an empty for loop event - serde_json::from_str("[]").unwrap() - }, - stop_early_override: None, - w_id: flow_job.workspace_id.clone(), - worker_dir: worker_dir.to_string(), - token: client.token.clone(), - }) - .await - .map_err(|e| { - Error::internal_err(format!( - "error sending update flow message to job completed channel: {e:#}" - )) - })?; - - return Ok(None); + return Ok(PushNextFlowJob::Done(Some(UpdateFlow { + flow: flow_job.id, + success: true, + result: if flow.modules.is_empty() { + to_raw_value(arc_flow_job_args.as_ref()) + } else { + // it has to be an empty for loop event + serde_json::from_str("[]").unwrap() + }, + stop_early_override: None, + w_id: flow_job.workspace_id.clone(), + worker_dir: worker_dir.to_string(), + token: client.token.clone(), + }))); } if matches!(step, Step::Step(0)) { @@ -1684,6 +1704,7 @@ async fn push_next_flow_job( flow_job.workspace_id.as_str() ) .fetch_one(db) + .warn_after_seconds(3) .await?; if no_flow_overlap { let overlapping = sqlx::query_scalar!( @@ -1704,6 +1725,7 @@ async fn push_next_flow_job( flow_job.runnable_path() ) .fetch_all(db) + .warn_after_seconds(3) .await?; if overlapping.len() > 0 { let overlapping_str = overlapping @@ -1711,27 +1733,21 @@ async fn push_next_flow_job( .map(|x| x.to_string()) .collect::>() .join(", "); - job_completed_tx - .send(SendResult::UpdateFlow { - flow: flow_job.id, - success: true, - result: serde_json::from_str( - &format!("\"not allowed to overlap with {overlapping_str}, scheduling next iteration\""), - ) - .unwrap(), - stop_early_override: Some(true), - w_id: flow_job.workspace_id.clone(), - worker_dir: worker_dir.to_string(), - token: client.token.clone(), - }) - .await - .map_err(|e| { - Error::internal_err(format!( - "error sending update flow message to job completed channel: {e:#}" - )) - })?; - return Ok(None); + return Ok(PushNextFlowJob::Done(Some( + UpdateFlow { + flow: flow_job.id, + success: true, + result: serde_json::from_str( + &format!("\"not allowed to overlap with {overlapping_str}, scheduling next iteration\""), + ) + .unwrap(), + stop_early_override: Some(true), + w_id: flow_job.workspace_id.clone(), + worker_dir: worker_dir.to_string(), + token: client.token.clone(), + } + ))); } } } @@ -1749,26 +1765,18 @@ async fn push_next_flow_job( flow_job.scheduled_for.to_string(), )]), ) + .warn_after_seconds(3) .await?; if skip { - job_completed_tx - .send(SendResult::UpdateFlow { - flow: flow_job.id, - success: true, - result: serde_json::from_str("\"stopped early\"").unwrap(), - stop_early_override: Some(true), - w_id: flow_job.workspace_id.clone(), - worker_dir: worker_dir.to_string(), - token: client.token.clone(), - }) - .await - .map_err(|e| { - Error::internal_err(format!( - "error sending update flow message to job completed channel: {e:#}" - )) - })?; - - return Ok(None); + return Ok(PushNextFlowJob::Done(Some(UpdateFlow { + flow: flow_job.id, + success: true, + result: serde_json::from_str("\"stopped early\"").unwrap(), + stop_early_override: Some(true), + w_id: flow_job.workspace_id.clone(), + worker_dir: worker_dir.to_string(), + token: client.token.clone(), + }))); } } } @@ -1787,7 +1795,10 @@ async fn push_next_flow_job( if last_job_result.is_some() { last_job_result.unwrap() } else { - match get_previous_job_result(db, flow_job.workspace_id.as_str(), &status).await? { + match get_previous_job_result(db, flow_job.workspace_id.as_str(), &status) + .warn_after_seconds(3) + .await? + { None => Arc::new(to_raw_value(&json!("{}"))), Some(previous_job_result) => Arc::new(previous_job_result), } @@ -1806,7 +1817,7 @@ async fn push_next_flow_job( FlowStatusModule::WaitingForPriorSteps { .. } | FlowStatusModule::WaitingForEvents { .. } ) { if let Some((suspend, last)) = needs_resume(&flow, &status) { - let mut tx = db.begin().await?; + let mut tx = db.begin().warn_after_seconds(3).await?; /* Lock this row to prevent the suspend column getting out out of sync * if a resume message arrives after we fetch and count them here. @@ -1817,6 +1828,7 @@ async fn push_next_flow_job( flow_job.id ) .fetch_one(&mut *tx) + .warn_after_seconds(3) .await .context("lock flow in queue")?; @@ -1825,7 +1837,9 @@ async fn push_next_flow_job( ) .bind(last) .fetch_all(&mut *tx) - .await? + .warn_after_seconds(3) + .await + ? .into_iter() .collect::>(); @@ -1865,6 +1879,7 @@ async fn push_next_flow_job( None, None ) + .warn_after_seconds(3) .await .map_err(|e| { Error::ExecutionErr(format!( @@ -1897,6 +1912,7 @@ async fn push_next_flow_job( flow_job.id ) .execute(&mut *tx) + .warn_after_seconds(3) .await?; } @@ -1938,6 +1954,7 @@ async fn push_next_flow_job( Some(&serde_json::json!({"approved": false, "job_id": flow_job.id, "details": "Suspend timed out without approval but can continue".to_string()}).to_string()), None, ) + .warn_after_seconds(3) .await?; } @@ -1959,6 +1976,7 @@ async fn push_next_flow_job( flow_job.id ) .execute(&mut *tx) + .warn_after_seconds(3) .await?; // Remove the approval conditions from the flow status @@ -1969,10 +1987,11 @@ async fn push_next_flow_job( flow_job.id ) .execute(&mut *tx) + .warn_after_seconds(3) .await?; /* continue on and run this job! */ - tx.commit().await?; + tx.commit().warn_after_seconds(3).await?; /* not enough messages to do this job, "park"/suspend until there are */ } else if matches!( @@ -2002,6 +2021,7 @@ async fn push_next_flow_job( flow_job.id, ) .execute(&mut *tx) + .warn_after_seconds(3) .await?; sqlx::query!( @@ -2010,10 +2030,11 @@ async fn push_next_flow_job( flow_job.id, ) .execute(&mut *tx) + .warn_after_seconds(3) .await?; - tx.commit().await?; - return Ok(None); + tx.commit().warn_after_seconds(3).await?; + return Ok(PushNextFlowJob::Done(None)); /* cancelled or we're WaitingForEvents but we don't have enough messages (timed out) */ } else { @@ -2027,9 +2048,10 @@ async fn push_next_flow_job( Some(&serde_json::json!({"approved": false, "job_id": flow_job.id, "details": "Suspend timed out without approval and is cancelled".to_string()}).to_string()), None, ) + .warn_after_seconds(3) .await?; } - tx.commit().await?; + tx.commit().warn_after_seconds(3).await?; let (logs, error_name) = if let Some(disapprover) = is_disapproved { ( @@ -2057,26 +2079,18 @@ async fn push_next_flow_job( logs.clone(), &db.into(), ) + .warn_after_seconds(3) .await; - job_completed_tx - .send(SendResult::UpdateFlow { - flow: flow_job.id, - success: false, - result: to_raw_value(&result), - stop_early_override: None, - w_id: flow_job.workspace_id.clone(), - worker_dir: worker_dir.to_string(), - token: client.token.clone(), - }) - .await - .map_err(|e| { - Error::internal_err(format!( - "error sending update flow message to job completed channel: {e:#}" - )) - })?; - - return Ok(None); + return Ok(PushNextFlowJob::Done(Some(UpdateFlow { + flow: flow_job.id, + success: false, + result: to_raw_value(&result), + stop_early_override: None, + w_id: flow_job.workspace_id.clone(), + worker_dir: worker_dir.to_string(), + token: client.token.clone(), + }))); } } } @@ -2137,6 +2151,7 @@ async fn push_next_flow_job( None, None, ) + .warn_after_seconds(3) .await .map_err(|e| { Error::ExecutionErr(format!( @@ -2237,6 +2252,7 @@ async fn push_next_flow_job( flow_job.id ) .execute(db) + .warn_after_seconds(3) .await .context("update flow retry")?; }; @@ -2255,7 +2271,9 @@ async fn push_next_flow_job( drop(resume_messages); let is_skipped = if let Some(skip_if) = &module.skip_if { - let idcontext = get_transform_context(&flow_job, previous_id.as_str(), &status).await?; + let idcontext = get_transform_context(&flow_job, previous_id.as_str(), &status) + .warn_after_seconds(3) + .await?; compute_bool_from_expr( &skip_if.expr, arc_flow_job_args.clone(), @@ -2266,6 +2284,7 @@ async fn push_next_flow_job( Some((resumes.clone(), resume.clone(), approvers.clone())), None, ) + .warn_after_seconds(3) .await? } else { false @@ -2295,6 +2314,7 @@ async fn push_next_flow_job( &flow_job.workspace_id ) .fetch_optional(db) + .warn_after_seconds(3) .await?; if let Some(args) = args { Ok(Marc::new(args.map(|x| x.0).unwrap_or_else(HashMap::new))) @@ -2327,7 +2347,9 @@ async fn push_next_flow_job( | FlowModuleValue::FlowScript { input_transforms, .. } | FlowModuleValue::Flow { input_transforms, .. }, ) => { - let ctx = get_transform_context(&flow_job, &previous_id, &status).await?; + let ctx = get_transform_context(&flow_job, &previous_id, &status) + .warn_after_seconds(3) + .await?; transform_context = Some(ctx); let by_id = transform_context.as_ref().unwrap(); transform_input( @@ -2340,6 +2362,7 @@ async fn push_next_flow_job( by_id, client, ) + .warn_after_seconds(3) .await .map(Marc::new) } @@ -2370,6 +2393,7 @@ async fn push_next_flow_job( approvers.clone(), is_skipped, ) + .warn_after_seconds(3) .await?; tracing::info!(id = %flow_job.id, root_id = %job_root, "next flow transform computed"); @@ -2395,6 +2419,7 @@ async fn push_next_flow_job( flow_job.id ) .fetch_optional(db) + .warn_after_seconds(3) .await? .flatten(); @@ -2404,13 +2429,13 @@ async fn push_next_flow_job( if let Some(status) = status { // // flow is reprocessed by the worker in a state where the module has completed successfully. - return Ok(Some(PushNextFlowJobRec { + return Ok(PushNextFlowJob::Rec(PushNextFlowJobRec { flow_job: flow_job, status: status, })); } else { return Err(Error::BadRequest( - "impossible to parse new flow status after applying innr flows".to_string(), + "impossible to parse new flow status after applying inner flows".to_string(), )); } } @@ -2433,7 +2458,7 @@ async fn push_next_flow_job( }; let len = job_payloads.len(); - let mut tx = db.begin().await?; + let mut tx = db.begin().warn_after_seconds(3).await?; let nargs = args.as_ref(); for (i, payload_tag) in job_payloads.into_iter().enumerate() { if i % 100 == 0 && i != 0 { @@ -2443,6 +2468,7 @@ async fn push_next_flow_job( flow_job.id, ) .execute(db) + .warn_after_seconds(3) .await?; } tracing::debug!(id = %flow_job.id, root_id = %job_root, "pushing job {i} of {len}"); @@ -2481,7 +2507,9 @@ async fn push_next_flow_job( args.insert("iter".to_string(), to_raw_value(new_args)); if let Some(input_transforms) = simple_input_transforms { //previous id is none because we do not want to use previous id if we are in a for loop - let ctx = get_transform_context(&flow_job, "", &status).await?; + let ctx = get_transform_context(&flow_job, "", &status) + .warn_after_seconds(3) + .await?; let ti = transform_input( Marc::new(args), arc_last_job_result.clone(), @@ -2492,6 +2520,7 @@ async fn push_next_flow_job( &ctx, client, ) + .warn_after_seconds(3) .await .map_err(|e| { Error::ExecutionErr( @@ -2527,7 +2556,9 @@ async fn push_next_flow_job( to_raw_value(&json!({ "index": i as i32, "value": itered[i]})), ); if let Some(input_transforms) = simple_input_transforms { - let ctx = get_transform_context(&flow_job, &previous_id, &status).await?; + let ctx = get_transform_context(&flow_job, &previous_id, &status) + .warn_after_seconds(3) + .await?; let ti = transform_input( Marc::new(hm), arc_last_job_result.clone(), @@ -2538,6 +2569,7 @@ async fn push_next_flow_job( &ctx, client, ) + .warn_after_seconds(3) .await .map_err(|e| { Error::ExecutionErr(format!( @@ -2608,6 +2640,7 @@ async fn push_next_flow_job( flow_job.workspace_id, ) .fetch_optional(&mut *tx) + .warn_after_seconds(3) .await? .map(|x| x.into()) } else { @@ -2668,6 +2701,7 @@ async fn push_next_flow_job( worker_name ) .execute(&mut *inner_tx) + .warn_after_seconds(3) .await; } @@ -2688,6 +2722,7 @@ async fn push_next_flow_job( uuid, ) .execute(&mut *inner_tx) + .warn_after_seconds(3) .await?; } tracing::debug!(id = %flow_job.id, root_id = %job_root, "updated suspend for {uuid}"); @@ -2707,6 +2742,7 @@ async fn push_next_flow_job( root_job.unwrap_or(flow_job.id) ) .execute(&mut *inner_tx) + .warn_after_seconds(3) .await?; } @@ -2731,6 +2767,7 @@ async fn push_next_flow_job( uuid ) .execute(&mut *tx) + .warn_after_seconds(3) .await?; tracing::debug!(id = %flow_job.id, root_id = %job_root, "updated parallel monitor lock for {uuid}"); } @@ -2844,6 +2881,7 @@ async fn push_next_flow_job( flow_job.id ) .execute(&mut *tx) + .warn_after_seconds(3) .await?; } Step::PreprocessorStep => { @@ -2860,6 +2898,7 @@ async fn push_next_flow_job( flow_job.id ) .execute(&mut *tx) + .warn_after_seconds(3) .await?; } Step::Step(i) => { @@ -2877,6 +2916,7 @@ async fn push_next_flow_job( flow_job.id ) .execute(&mut *tx) + .warn_after_seconds(3) .await?; } }; @@ -2888,6 +2928,7 @@ async fn push_next_flow_job( flow_job.id ) .execute(&mut *tx) + .warn_after_seconds(3) .await?; if continue_on_same_worker { @@ -2903,10 +2944,11 @@ async fn push_next_flow_job( if continue_on_same_worker { same_worker_tx .send(SameWorkerPayload { job_id: first_uuid, recoverable: true }) + .warn_after_seconds(3) .await .map_err(to_anyhow)?; } - return Ok(None); + return Ok(PushNextFlowJob::Done(None)); } // async fn jump_to_next_step( diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index a3b195de7a..67140a5fbd 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.490.0"; +export const VERSION = "v1.491.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index 9be55b1c60..100fc521bc 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -63,7 +63,7 @@ export { // } // }); -export const VERSION = "1.490.0"; +export const VERSION = "1.491.1"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 1fa3ad37d4..95223be70f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.490.0", + "version": "1.491.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.490.0", + "version": "1.491.1", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index fdfeda2e4c..1dd17df57e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.490.0", + "version": "1.491.1", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/frontend/src/lib/components/apps/components/display/table/AppAggridTable.svelte b/frontend/src/lib/components/apps/components/display/table/AppAggridTable.svelte index 16fea27331..484f11133a 100644 --- a/frontend/src/lib/components/apps/components/display/table/AppAggridTable.svelte +++ b/frontend/src/lib/components/apps/components/display/table/AppAggridTable.svelte @@ -303,8 +303,8 @@ function getIdFromData(data: any): string { return resolvedConfig?.rowIdCol && resolvedConfig?.rowIdCol != '' - ? (data?.[resolvedConfig?.rowIdCol] ?? data['__index']) - : data['__index'] + ? (data?.[resolvedConfig?.rowIdCol] ?? data?.['__index']) + : data?.['__index'] } function mountGrid() { @@ -389,8 +389,24 @@ e.api.deselectAll() outputs?.selectedRow?.set({}) outputs?.selectedRowIndex.set(0) - } else { - e.api.getRowNode(index.toString())?.setSelected(true) + } else if (Array.isArray(index)) { + // select all rows matching the indixes + e.api.deselectAll() + index.forEach((i) => { + let rowId = getIdFromData(value[i]) + if (rowId) { + e.api.getRowNode(rowId)?.setSelected(true, false) + } + }) + } else if (typeof index === 'number') { + let rowId = getIdFromData(value[index]) + if (rowId) { + e.api.getRowNode(rowId)?.setSelected(true, true) + outputs?.selectedRowIndex.set(index) + const row = { ...value[index] } + delete row['__index'] + outputs?.selectedRow?.set(row) + } } }, setValue(nvalue) { diff --git a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte index 650f951935..9dd3c9860a 100644 --- a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte +++ b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte @@ -55,7 +55,7 @@ bun: 'TypeScript', php: 'PHP', rust: 'Rust', - ansible: 'Ansible Playbook', + ansible: 'Ansible', csharp: 'C#', nu: 'Nu', java: 'Java' diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index 3d58eac5b7..fd7a42e587 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -249,6 +249,7 @@ export async function main(message: string, name: string, step_id: string) { ` const POSTGRES_INIT_CODE = `-- to pin the database use '-- database f/your/path' +-- to stream a large query result to your workspace storage use '-- s3' -- to only return the result of the last query use '--return_last_result' -- $1 name1 = default arg -- $2 name2 @@ -259,6 +260,7 @@ UPDATE demo SET col2 = \$4::INT WHERE col2 = \$2::INT; ` const MYSQL_INIT_CODE = `-- to pin the database use '-- database f/your/path' +-- to stream a large query result to your workspace storage use '-- s3' -- :name1 (text) = default arg -- :name2 (int) -- :name3 (int) @@ -267,6 +269,7 @@ UPDATE demo SET col2 = :name3 WHERE col2 = :name2; ` const BIGQUERY_INIT_CODE = `-- to pin the database use '-- database f/your/path' +-- to stream a large query result to your workspace storage use '-- s3' -- @name1 (string) = default arg -- @name2 (integer) -- @name3 (string[]) @@ -276,6 +279,7 @@ UPDATE \`demodb.demo\` SET col2 = @name4 WHERE col2 = @name2; ` const ORACLEDB_INIT_CODE = `-- to pin the database use '-- database f/your/path' +-- to stream a large query result to your workspace storage use '-- s3' -- :name1 (text) = default arg -- :name2 (int) -- :name3 (int) @@ -284,6 +288,7 @@ UPDATE demo SET col2 = :name3 WHERE col2 = :name2; ` const SNOWFLAKE_INIT_CODE = `-- to pin the database use '-- database f/your/path' +-- to stream a large query result to your workspace storage use '-- s3' -- ? name1 (varchar) = default arg -- ? name2 (int) INSERT INTO demo VALUES (?, ?); @@ -294,6 +299,7 @@ UPDATE demo SET col2 = ? WHERE col2 = ?; const MSSQL_INIT_CODE = `-- return_last_result -- to pin the database use '-- database f/your/path' +-- to stream a large query result to your workspace storage use '-- s3' -- @P1 name1 (varchar) = default arg -- @P2 name2 (int) -- @P3 name3 (int) diff --git a/frontend/src/lib/scripts.ts b/frontend/src/lib/scripts.ts index 61b19dcca4..bc8a44b434 100644 --- a/frontend/src/lib/scripts.ts +++ b/frontend/src/lib/scripts.ts @@ -136,7 +136,7 @@ const scriptLanguagesArray: [SupportedLanguage | 'docker' | 'bunnative', string] ['powershell', 'PowerShell'], ['php', 'PHP'], ['rust', 'Rust'], - ['ansible', 'Ansible Playbook'], + ['ansible', 'Ansible'], ['csharp', 'C#'], ['docker', 'Docker'], ['nu', 'Nu'], diff --git a/lsp/Pipfile b/lsp/Pipfile index 3381ba3d5f..7c7252a3a8 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.490.0" -wmill_pg = ">=1.490.0" +wmill = ">=1.491.1" +wmill_pg = ">=1.491.1" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 525cfea75b..a4c1530ea5 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.490.0 + version: 1.491.1 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 09bfa59f8e..561264500b 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.490.0' + ModuleVersion = '1.491.1' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index ad35e507cc..b568c2f2bf 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.490.0" +version = "1.491.1" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index d4d753a45a..13cdaa9052 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.490.0" +version = "1.491.1" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/build.jsr.sh b/typescript-client/build.jsr.sh index 06dae6605b..e203c17fdf 100755 --- a/typescript-client/build.jsr.sh +++ b/typescript-client/build.jsr.sh @@ -14,5 +14,5 @@ cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/build.sh b/typescript-client/build.sh index 5f9cdbc4ec..4934c5e673 100755 --- a/typescript-client/build.sh +++ b/typescript-client/build.sh @@ -4,7 +4,7 @@ script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" rm -rf "${script_dirpath}/src" -npx --yes @hey-api/openapi-ts@0.43.0 --input "${script_dirpath}/../backend/windmill-api/openapi.yaml" --output "${script_dirpath}/src" --useOptions --schemas false +npx --yes @hey-api/openapi-ts@0.43.0 --input "${script_dirpath}/../backend/windmill-api/openapi.yaml" --output "${script_dirpath}/src" --useOptions --schemas false cat < temp_file && mv temp_file src/core/OpenAPI.ts const getEnv = (key: string) => { if (typeof window === "undefined") { @@ -39,4 +39,4 @@ cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 258ea76215..25de187598 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -456,8 +456,8 @@ export async function getProgress(jobId?: any): Promise { } /** - * Set a flow user state - * @param key key of the state + * Set a flow user state + * @param key key of the state * @param value value of the state */ @@ -920,6 +920,15 @@ interface SlackApprovalOptions { dynamicEnumsJson?: Record; } +interface TeamsApprovalOptions { + teamName: string; + channelName: string; + message?: string; + approver?: string; + defaultArgsJson?: Record; + dynamicEnumsJson?: Record; +} + /** * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. * @@ -1012,6 +1021,98 @@ export async function requestInteractiveSlackApproval({ }); } +/** + * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Teams approval request. + * @param {string} options.teamName - The Teams team name where the approval request will be sent. + * @param {string} options.channelName - The Teams channel name where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Teams approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Teams approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the `JobService.getTeamsApprovalPayload` call fails. + * + * **Usage Example:** + * ```typescript + * await requestInteractiveTeamsApproval({ + * teamName: "admins-teams", + * channelName: "admins-teams-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * ``` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +export async function requestInteractiveTeamsApproval({ + teamName, + channelName, + message, + approver, + defaultArgsJson, + dynamicEnumsJson, +}: TeamsApprovalOptions): Promise { + const workspace = getWorkspace(); + const flowJobId = getEnv("WM_FLOW_JOB_ID"); + + if (!flowJobId) { + throw new Error( + "You can't use this function in a standalone script or flow step preview. Please use it in a flow or a flow preview." + ); + } + + const flowStepId = getEnv("WM_FLOW_STEP_ID"); + if (!flowStepId) { + throw new Error("This function can only be called as a flow step"); + } + + // Only include non-empty parameters + const params: { + approver?: string; + message?: string; + teamName: string; + channelName: string; + flowStepId: string; + defaultArgsJson?: string; + dynamicEnumsJson?: string; + } = { + teamName, + channelName, + flowStepId, + }; + + if (message) { + params.message = message; + } + if (approver) { + params.approver = approver; + } + + if (defaultArgsJson) { + params.defaultArgsJson = JSON.stringify(defaultArgsJson); + } + + if (dynamicEnumsJson) { + params.dynamicEnumsJson = JSON.stringify(dynamicEnumsJson); + } + + await JobService.getTeamsApprovalPayload({ + workspace, + ...params, + id: getEnv("WM_JOB_ID") ?? "NO_JOB_ID", + }); +} + async function getMockedApi(): Promise { if (mockedApi) { return mockedApi; diff --git a/typescript-client/dev.nu b/typescript-client/dev.nu index 36b384eeb3..9bd61a7bd3 100755 --- a/typescript-client/dev.nu +++ b/typescript-client/dev.nu @@ -1,15 +1,15 @@ #! /usr/bin/env nu -let cache = "/tmp/windmill/cache_nomount/bun/" +let cache = "/tmp/windmill/cache_nomount/bun/" # Clean cache def "main clean" [] { - ^rm -rf ($cache ++ "/windmill-client") + ^rm -rf ($cache ++ "/windmill-client") } # Watch changes in directory and autopatch (watchexec required) def "main watch" [] { - # watchexec -w ../backend/windmill-api/openapi.yaml './dev.nu -g' & + # watchexec -w ../backend/windmill-api/openapi.yaml './dev.nu -g' & # TODO: Watch openapi.yaml ^watchexec ./dev.nu @@ -19,7 +19,7 @@ def "main watch" [] { # To build you will need nushell and tsc (typescript compiler) # If none arguments selected, all will be turned on # If any argument specified, all others will be disabled -def main [ +def main [ --gen(-g) # Generate code (OpenAPI codegen) --compile(-c) # Compile code (TS >> JS) --patch(-p) # Patch @@ -47,16 +47,16 @@ def main [ rm -rf ($cache ++ windmill-client@*/dist/*) # Delete all script bundles - # rm -rf /tmp/windmill/cache/bun/* + # rm -rf /tmp/windmill/cache_nomount/bun/* # Copy files from local ./dist to every wm-client version in cache ls ($cache ++ "windmill-client/") | each { - |i| + |i| - let path = $i | get name; - ^cp -r dist/* ($path ++ "/dist") + let path = $i | get name; + ^cp -r dist/* ($path ++ "/dist") } } - print Done! + print Done! } diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 66a0bbeda6..5b3484ada2 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.490.0", + "version": "1.491.1", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index f9622baa51..b4f65e6429 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.490.0", + "version": "1.491.1", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index ab8adf7473..74d12d26c2 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.490.0 +1.491.1