starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-07 08:02:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into hc/ai-flow-chat

Browse Source
This commit is contained in:
HugoCasa
2025-05-27 09:44:24 +02:00
parent e68961bf15 5dcefeff84
commit 9661f1360c
104 changed files with 3451 additions and 1787 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/aider-after-review.yaml
+13 -13
View File
@@ -17,10 +17,11 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_REPOSITORY: ${{ github.repository }}
REVIEWER: ${{ github.event.review.user.login }}
ORG_ACCESS_TOKEN: ${{ secrets.ORG_ACCESS_TOKEN }}
run: |
ORG="windmill-labs"
STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Authorization: token $GH_TOKEN" \
-H "Authorization: token $ORG_ACCESS_TOKEN" \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/orgs/$ORG/members/$REVIEWER")
@@ -59,27 +60,25 @@ jobs:
- name: Prepare prompt for Aider
id: prepare_prompt
shell: bash
env:
GITHUB_REPOSITORY: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
REVIEW_BODY: ${{ github.event.review.body }}
run: |
# Get PR review body
REVIEW_BODY="${{ github.event.review.body }}"
REVIEW_BODY_Q=$(printf '%q' "$REVIEW_BODY")
REVIEW_BODY_ESCAPED="${REVIEW_BODY//\\/\\\\}"
REVIEW_BODY_ESCAPED="${REVIEW_BODY_ESCAPED//\"/\\\"}"
PR_NUMBER="${{ github.event.pull_request.number }}"
# Get all PR review comments
ALL_REVIEW_COMMENTS=$(gh api \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
/repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/comments \
| jq '[.[] | {diff_hunk: .diff_hunk, path: .path, body: .body}]')
/repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/comments)
FORMATTED_COMMENTS=$(jq -r '[.[] | {diff_hunk: .diff_hunk, path: .path, body: .body}]' <<< "$ALL_REVIEW_COMMENTS")
BASE_PROMPT="Fix the following issues in the PR based on the review feedback. The review body is prepended with REVIEW. The review comments are prepended with REVIEW_COMMENTS. The review body and comments are separated by a blank line."
printf -v COMPLETE_PROMPT "%s\nREVIEW:\n%s\nREVIEW_COMMENTS:\n%s" \
"$BASE_PROMPT" "$REVIEW_BODY_Q" "$ALL_REVIEW_COMMENTS"
echo "$COMPLETE_PROMPT"
COMPLETE_PROMPT="${BASE_PROMPT}"$'\n'"REVIEW:"$'\n'"${REVIEW_BODY_ESCAPED}"$'\n'"REVIEW_COMMENTS:"$'\n'"${FORMATTED_COMMENTS}"
# Use the proper multi-line output format
echo "prompt_content<<EOF" >> $GITHUB_OUTPUT
echo "$COMPLETE_PROMPT" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
@@ -91,4 +90,5 @@ jobs:
with:
needs_processing: false
base_prompt: ${{ needs.check-and-prepare.outputs.prompt_content }}
rules_files: "CLAUDE.md backend/CLAUDE.md frontend/CLAUDE.md"
secrets: inherit
.github/workflows/aider-common.yml
+121 -79
View File
@@ -33,7 +33,11 @@ on:
description: "Prompt for probe-chat"
required: false
type: string
default: 'I''m giving you a request that needs to be implemented. Your role is ONLY to give me the files that are relevant to the request and nothing else. The request is prepended with the word REQUEST. REQUEST: $FINAL_PROMPT. Give me all the files relevant to this request. Your output MUST be a single json array that can be parsed with programatic json parsing, with the relevant files. Files can be rust or typescript or javascript files. DO NOT INCLUDE ANY OTHER TEXT IN YOUR OUTPUT. ONLY THE JSON ARRAY. Example of output: ["file1.py", "file2.py"]'
default: 'I''m giving you a request that needs to be implemented. Your role is ONLY to give me the files that are relevant to the request and nothing else. The request is prepended with the word REQUEST. Give me all the files relevant to this request. Your output MUST be a single json array that can be parsed with programatic json parsing, with the relevant files. Files can be rust or typescript or javascript files. DO NOT INCLUDE ANY OTHER TEXT IN YOUR OUTPUT. ONLY THE JSON ARRAY. Example of output: ["file1.py", "file2.py"]'
rules_files:
description: "Rules files for Aider"
required: false
type: string
outputs:
files_to_edit:
description: "Files identified by probe-chat for editing"
@@ -67,6 +71,7 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
WINDMILL_TOKEN: ${{ secrets.WINDMILL_TOKEN }}
LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }}
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_AI_BOT_TOKEN }}
steps:
- name: Harden Runner
@@ -114,7 +119,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
python-version: "3.11"
- name: Cache Python dependencies
uses: actions/cache@v3
@@ -124,27 +129,18 @@ jobs:
restore-keys: |
${{ runner.os }}-pip-
- name: Cache Aider installation
id: cache-aider
uses: actions/cache@v3
with:
path: ~/.local/bin/aider
key: ${{ runner.os }}-aider-install-${{ hashFiles('**/requirements.txt', '**/setup.py') }}
restore-keys: |
${{ runner.os }}-aider-install-
- name: Install Aider and Dependencies
run: |
if [ -f ~/.local/bin/aider ] && [ -x ~/.local/bin/aider ]; then
echo "Using cached Aider installation"
export PATH="$HOME/.local/bin:$PATH"
else
echo "Installing Aider..."
python -m pip install aider-install; aider-install
fi
pip install -U google-generativeai
echo "Installing Aider..."
python -m pip install uv
python -m venv ~/uv-env
source ~/uv-env/bin/activate
uv pip install configargparse==1.7
uv pip install aider-chat==0.83.1
uv pip install -U google-generativeai
sudo apt-get update && sudo apt-get install -y jq
echo "$HOME/.local/bin" >> $GITHUB_PATH
echo "VIRTUAL_ENV_PATH=$HOME/uv-env" >> $GITHUB_ENV
- name: Create Prompt for Aider
id: create_prompt
@@ -206,7 +202,7 @@ jobs:
fi
else
echo "No issue title or body given. Using base prompt."
FINAL_PROMPT_CONTENT="$BASE_PROMPT_ENV"
FINAL_PROMPT_CONTENT=$(printf "%s\nINSTRUCTION:\n%s" "$BASE_PROMPT_ENV" "$INSTRUCTION_ENV")
fi
echo "Final prompt: $FINAL_PROMPT_CONTENT"
@@ -219,11 +215,11 @@ jobs:
shell: bash
env:
FINAL_PROMPT: ${{ steps.create_prompt.outputs.final_prompt }}
PROBE_PROMPT: ${{ inputs.probe_prompt }}
run: |
echo "Running probe-chat to find relevant files..."
# escape the final prompt
printf -v MESSAGE_FOR_PROBE 'I'\''m giving you a request that needs to be implemented. Your role is ONLY to give me the files that are relevant to the request and nothing else. The request is prepended with the word REQUEST.\nREQUEST: %s. Give me all the files relevant to this request. Your output MUST be a single json array that can be parsed with programatic json parsing, with the relevant files. Files can be rust or typescript or javascript files. DO NOT INCLUDE ANY OTHER TEXT IN YOUR OUTPUT. ONLY THE JSON ARRAY. Example of output: ["file1.py", "file2.py"]' "$FINAL_PROMPT"
MESSAGE_FOR_PROBE=$(printf "%s\nREQUEST:\n%s" "$PROBE_PROMPT" "$FINAL_PROMPT")
set -o pipefail
PROBE_OUTPUT=$(npx --yes @buger/probe-chat@latest --max-iterations 50 --model-name gemini-2.5-pro-preview-05-06 --message "$MESSAGE_FOR_PROBE") || {
@@ -256,24 +252,66 @@ jobs:
restore-keys: |
${{ runner.os }}-aider-
- name: Prepare branch for Aider
id: prepare_branch
env:
ISSUE_ID: ${{ inputs.issue_id }}
run: |
if [[ "$ISSUE_ID" != "" ]]; then
BRANCH_NAME="aider-fix-issue-${ISSUE_ID}"
# Check if branch exists remotely
if git ls-remote --heads origin $BRANCH_NAME | grep -q $BRANCH_NAME; then
echo "Branch $BRANCH_NAME already exists remotely, fetching it"
git fetch origin $BRANCH_NAME
git checkout $BRANCH_NAME
git pull origin $BRANCH_NAME
else
echo "Creating new branch $BRANCH_NAME"
git checkout -b $BRANCH_NAME
fi
echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT
else
# We're in a pull_request_review event
PR_NUMBER="${{ github.event.pull_request.number }}"
PR_HEAD_REF="${{ github.event.pull_request.head.ref }}"
echo "Handling pull_request_review for PR #$PR_NUMBER on branch $PR_HEAD_REF"
# Ensure we're on the correct branch
git config pull.rebase true
git fetch origin $PR_HEAD_REF
git checkout $PR_HEAD_REF
git pull origin $PR_HEAD_REF
echo "Using PR branch $PR_HEAD_REF for PR #$PR_NUMBER"
echo "BRANCH_NAME=$PR_HEAD_REF" >> $GITHUB_OUTPUT
fi
- name: Run Aider
id: run_aider
shell: bash
env:
FILES_TO_EDIT: ${{ steps.probe_files.outputs.files_to_edit }}
FINAL_PROMPT: ${{ steps.create_prompt.outputs.final_prompt }}
RULES_FILES: ${{ inputs.rules_files }}
run: |
source $VIRTUAL_ENV_PATH/bin/activate
echo "$FINAL_PROMPT" > .aider_final_prompt.txt
echo "FILES_TO_EDIT: $FILES_TO_EDIT"
RULES=""
if [ -n "$RULES_FILES" ]; then
for rule in $RULES_FILES; do
RULES="$RULES --read $rule"
done
fi
aider \
--read .cursor/rules/rust-best-practices.mdc \
--read .cursor/rules/svelte5-best-practices.mdc \
--read .cursor/rules/windmill-overview.mdc \
$RULES \
$FILES_TO_EDIT \
--model gemini/gemini-2.5-pro-preview-05-06 \
--message "create a test file in backend/test.txt with hello world in it" \
--message-file .aider_final_prompt.txt \
--yes \
--no-check-update \
--auto-commits \
@@ -295,40 +333,31 @@ jobs:
id: commit_and_push
env:
ISSUE_ID: ${{ inputs.issue_id }}
BRANCH_NAME: ${{ steps.prepare_branch.outputs.BRANCH_NAME }}
run: |
if [[ "$ISSUE_ID" != "" ]]; then
BRANCH_NAME="aider-fix-issue-${ISSUE_ID}"
# Check if branch exists remotely
if git ls-remote --heads origin $BRANCH_NAME | grep -q $BRANCH_NAME; then
echo "Branch $BRANCH_NAME already exists remotely, fetching it"
git fetch origin $BRANCH_NAME
git checkout $BRANCH_NAME
git pull origin $BRANCH_NAME
else
echo "Creating new branch $BRANCH_NAME"
git checkout -b $BRANCH_NAME
# Check if there are any uncommitted changes
if [[ -n $(git status --porcelain) ]]; then
echo "Found uncommitted changes, committing them"
git add .
git commit -m "Aider changes"
fi
echo "Created/checked out branch $BRANCH_NAME for issue #${ISSUE_ID}"
git push origin $BRANCH_NAME
echo "Pushed to branch $BRANCH_NAME"
echo "PR_BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT
echo "CHANGES_APPLIED_MESSAGE=Aider changes pushed to branch $BRANCH_NAME." >> $GITHUB_OUTPUT
# Push changes to the branch
if git push origin $BRANCH_NAME; then
echo "Pushed to branch $BRANCH_NAME"
echo "PR_BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT
echo "CHANGES_APPLIED_MESSAGE=Aider changes pushed to branch $BRANCH_NAME." >> $GITHUB_OUTPUT
echo "CHANGES_APPLIED=true" >> $GITHUB_OUTPUT
else
echo "::warning::Push to PR branch $BRANCH_NAME failed."
echo "CHANGES_APPLIED_MESSAGE=Aider ran, but failed to push changes to PR branch $BRANCH_NAME." >> $GITHUB_OUTPUT
echo "CHANGES_APPLIED=false" >> $GITHUB_OUTPUT
fi
else
# We're in a pull_request_review event
PR_NUMBER="${{ github.event.pull_request.number }}"
PR_HEAD_REF="${{ github.event.pull_request.head.ref }}"
echo "Handling pull_request_review for PR #$PR_NUMBER on branch $PR_HEAD_REF"
# Ensure we're on the correct branch
git config pull.rebase true
git fetch origin $PR_HEAD_REF
git checkout $PR_HEAD_REF
git pull origin $PR_HEAD_REF
echo "Attempting to push changes to PR branch $PR_HEAD_REF for PR #$PR_NUMBER"
echo "Attempting to push changes to PR branch $PR_HEAD_REF"
if git push origin $PR_HEAD_REF; then
echo "Push to $PR_HEAD_REF successful (or no new changes to push)."
echo "CHANGES_APPLIED_MESSAGE=Aider changes (if any) pushed to PR branch $PR_HEAD_REF." >> $GITHUB_OUTPUT
@@ -349,23 +378,20 @@ jobs:
PR_BRANCH: ${{ steps.commit_and_push.outputs.PR_BRANCH_NAME }}
ISSUE_NUM: ${{ inputs.issue_id }}
ISSUE_TITLE: ${{ inputs.issue_title }}
GITHUB_EVENT_NAME: ${{ github.event_name }}
run: |
# Debug: Check latest commit and branch status
echo "Checking latest commit on branch $PR_BRANCH"
git log -1 --pretty=format:"%h - %an, %ar : %s"
echo "Changes not yet committed:"
git status --porcelain
# Check if there are any changes to commit
if [[ -n $(git status --porcelain) ]]; then
echo "Found uncommitted changes, committing them"
git add .
git commit -m "Aider changes for issue #${ISSUE_NUM}"
git push origin $PR_BRANCH
fi
# Create PR description in a temporary file to avoid command line length limits and ensure it stays under 40k chars
HEADER="This PR was created automatically by Aider to fix issue #${ISSUE_NUM}."
# if event is repository_dispatch, add the issue title to the header
if [ "$GITHUB_EVENT_NAME" == "repository_dispatch" ]; then
if [[ "${{ github.event.client_payload.source }}" == "linear" ]]; then
HEADER="This PR was created automatically by Aider to fix issue #linear:${ISSUE_NUM}."
elif [[ "${{ github.event.client_payload.source }}" == "discord" ]]; then
HEADER="This PR was created automatically by Aider to fix issue #discord:${ISSUE_NUM}."
fi
fi
cat > /tmp/pr-description.md << EOL | head -c 40000
This PR was created automatically by Aider to fix issue #${ISSUE_NUM}.
$HEADER
## Aider Output
\`\`\`
@@ -375,11 +401,16 @@ jobs:
# Create PR using the file for the body content, handle errors gracefully
set +e # Don't exit on error
PR_TITLE="[Aider PR] Fix: ${ISSUE_TITLE}"
if [ -z "$ISSUE_TITLE" ]; then
PR_TITLE="[Aider PR] AI changes after request"
fi
gh pr create \
--title "[Aider PR] Fix: ${ISSUE_TITLE}" \
--title "$PR_TITLE" \
--body-file /tmp/pr-description.md \
--head "$PR_BRANCH" \
--base main
--base main \
--draft
PR_CREATE_EXIT_CODE=$?
set -e # Re-enable exit on error
@@ -437,12 +468,13 @@ jobs:
GITHUB_REPOSITORY: ${{ github.repository }}
JOB_STATUS: ${{ job.status }}
PR_CREATED: ${{ steps.create_pr.outputs.PR_CREATED }}
PR_URL: ${{ steps.create_pr.outputs.PR_URL }}
run: |
echo "Commenting on issue/PR #${{ github.event.issue.number }} to let the user know Aider has finished working on the request."
if [[ "$JOB_STATUS" == "success" ]]; then
if [[ "$PR_CREATED" == "true" ]]; then
COMMENT_BODY="🤖 Aider has finished working on your request. A PR has been created."
COMMENT_BODY="🤖 Aider has finished working on your request. A PR has been created. $PR_URL"
else
COMMENT_BODY="🤖 Aider has finished working on your request, but was unable to create a PR."
fi
@@ -460,12 +492,14 @@ jobs:
JOB_STATUS: ${{ job.status }}
LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }}
PR_CREATED: ${{ steps.create_pr.outputs.PR_CREATED }}
PR_URL: ${{ steps.create_pr.outputs.PR_URL }}
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_AI_BOT_TOKEN }}
SOURCE: ${{ github.event.client_payload.source }}
run: |
echo "Commenting on linear issue #${{ github.event.client_payload.issue_id }} to let the user know Aider has finished working on the request."
echo "Notifying user about Aider completion status for $SOURCE request #${{ github.event.client_payload.issue_id }}"
if [[ "$JOB_STATUS" == "success" ]]; then
if [[ "$PR_CREATED" == "true" ]]; then
COMMENT_BODY="🤖 Aider has finished working on your request. A PR has been created."
COMMENT_BODY="🤖 Aider has finished working on your request. A PR has been created. $PR_URL"
else
COMMENT_BODY="🤖 Aider has finished working on your request, but was unable to create a PR."
fi
@@ -473,8 +507,16 @@ jobs:
COMMENT_BODY="⚠️ Aider encountered issues while working on your request. Please check the workflow logs for details."
fi
curl -X POST \
-H "Authorization: $LINEAR_API_KEY" \
-H "Content-Type: application/json" \
"https://api.linear.app/graphql" \
-d "{\"query\":\"mutation { commentCreate(input: { issueId: \\\"${{ github.event.client_payload.issue_id }}\\\", body: \\\"${COMMENT_BODY}\\\" }) { success } }\"}"
if [[ "$SOURCE" == "discord" ]]; then
curl -X POST \
-H "Authorization: Bot $DISCORD_BOT_TOKEN" \
-H "Content-Type: application/json" \
"https://discord.com/api/v10/channels/${{ github.event.client_payload.channel_id }}/messages" \
-d "{\"content\":\"${COMMENT_BODY}\"}"
else
curl -X POST \
-H "Authorization: $LINEAR_API_KEY" \
-H "Content-Type: application/json" \
"https://api.linear.app/graphql" \
-d "{\"query\":\"mutation { commentCreate(input: { issueId: \\\"${{ github.event.client_payload.issue_id }}\\\", body: \\\"${COMMENT_BODY}\\\" }) { success } }\"}"
fi
.github/workflows/aider-external.yaml
+80
View File
@@ -0,0 +1,80 @@
name: External Aider Issue Fix
on:
repository_dispatch:
types: [external_issue_fix]
jobs:
check-and-prepare:
runs-on: ubicloud-standard-2
permissions:
contents: write
pull-requests: write
outputs:
issue_title: ${{ steps.determine_inputs.outputs.ISSUE_TITLE }}
issue_body: ${{ steps.determine_inputs.outputs.ISSUE_BODY }}
instruction: ${{ steps.determine_inputs.outputs.INSTRUCTION }}
env:
GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
WINDMILL_TOKEN: ${{ secrets.WINDMILL_TOKEN }}
LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }}
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_AI_BOT_TOKEN }}
steps:
- name: Acknowledge Request
env:
LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }}
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_AI_BOT_TOKEN }}
run: |
if [[ "${{ github.event.client_payload.source }}" == "linear" ]]; then
echo "Commenting on Linear issue #${{ github.event.client_payload.issue_id }} to acknowledge the request."
curl -X POST \
-H "Authorization: $LINEAR_API_KEY" \
-H "Content-Type: application/json" \
"https://api.linear.app/graphql" \
-d "{\"query\":\"mutation { commentCreate(input: { issueId: \\\"${{ github.event.client_payload.issue_id }}\\\", body: \\\"🤖 Aider is starting to work on your request. I'll update you here once I have a PR ready. Please be patient, this might take a few minutes.\\\" }) { success } }\"}"
elif [[ "${{ github.event.client_payload.source }}" == "discord" ]]; then
echo "Commenting on Discord thread #${{ github.event.client_payload.channel_id }} to acknowledge the request."
curl -X POST \
-H "Authorization: Bot $DISCORD_BOT_TOKEN" \
-H "Content-Type: application/json" \
"https://discord.com/api/v10/channels/${{ github.event.client_payload.channel_id }}/messages" \
-d "{\"content\":\"🤖 Aider is starting to work on your request. I'll update you here once I have a PR ready. Please be patient, this might take a few minutes.\"}"
fi
- name: Determine inputs for Aider
id: determine_inputs
shell: bash
env:
ISSUE_TITLE: ${{ github.event.client_payload.issue_title }}
ISSUE_BODY: ${{ github.event.client_payload.issue_body }}
INSTRUCTION: ${{ github.event.client_payload.instruction }}
run: |
echo "Determining inputs for Aider..."
echo "ISSUE_TITLE<<EOF_AIDER_TITLE" >> "$GITHUB_OUTPUT"
echo "$ISSUE_TITLE" >> "$GITHUB_OUTPUT"
echo "EOF_AIDER_TITLE" >> "$GITHUB_OUTPUT"
echo "ISSUE_BODY<<EOF_AIDER_BODY" >> "$GITHUB_OUTPUT"
echo "$ISSUE_BODY" >> "$GITHUB_OUTPUT"
echo "EOF_AIDER_BODY" >> "$GITHUB_OUTPUT"
echo "INSTRUCTION<<EOF_AIDER_INSTRUCTION" >> "$GITHUB_OUTPUT"
echo "$INSTRUCTION" >> "$GITHUB_OUTPUT"
echo "EOF_AIDER_INSTRUCTION" >> "$GITHUB_OUTPUT"
echo "Finished determining inputs."
run-aider:
needs: check-and-prepare
uses: ./.github/workflows/aider-common.yml
with:
issue_title: ${{ needs.check-and-prepare.outputs.issue_title }}
issue_body: ${{ needs.check-and-prepare.outputs.issue_body }}
instruction: ${{ needs.check-and-prepare.outputs.instruction }}
issue_id: ${{ github.event.client_payload.issue_id }}
rules_files: "CLAUDE.md backend/CLAUDE.md frontend/CLAUDE.md"
secrets: inherit
.github/workflows/aider.yaml
+44 -22
View File
@@ -20,10 +20,11 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_REPOSITORY: ${{ github.repository }}
COMMENTER: ${{ github.event.comment.user.login }}
ORG_ACCESS_TOKEN: ${{ secrets.ORG_ACCESS_TOKEN }}
run: |
ORG="windmill-labs"
STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Authorization: token $GH_TOKEN" \
-H "Authorization: token $ORG_ACCESS_TOKEN" \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/orgs/$ORG/members/$COMMENTER")
@@ -66,6 +67,12 @@ jobs:
- name: Determine inputs for Aider
id: determine_inputs
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COMMENT_BODY: ${{ github.event.comment.body }}
ISSUE_NUMBER: ${{ github.event.issue.number }}
GITHUB_REPOSITORY: ${{ github.repository }}
LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }}
run: |
echo "Determining inputs for Aider..."
ISSUE_TITLE_VAL=""
@@ -73,28 +80,44 @@ jobs:
if [[ ! -z "${{ github.event.issue.pull_request }}" ]]; then
echo "This is a comment on a Pull Request"
PR_NUMBER="${{ github.event.issue.number }}"
PR_NUMBER="$ISSUE_NUMBER"
PR_BODY_JSON=$(GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" gh pr view "$PR_NUMBER" --json body --repo "$GITHUB_REPOSITORY")
PR_BODY_JSON=$(gh pr view "$PR_NUMBER" --json body --repo "$GITHUB_REPOSITORY")
if [[ $? -ne 0 ]]; then
echo "Error fetching PR body for PR #$PR_NUMBER"
PR_BODY_VAL=""
else
PR_BODY_VAL=$(echo "$PR_BODY_JSON" | jq -r .body)
PR_BODY_VAL=$(jq -r '.body // ""' <<< "$PR_BODY_JSON")
fi
if [[ ! -z "$PR_BODY_VAL" ]]; then
REFERENCED_ISSUE=$(echo "$PR_BODY_VAL" | grep -oE "#[0-9]+" | grep -oE "[0-9]+" | head -1)
if [[ ! -z "$REFERENCED_ISSUE" ]]; then
echo "Found referenced issue #$REFERENCED_ISSUE in PR description"
REFERENCED_ISSUE=""
if [[ "$PR_BODY_VAL" =~ \#linear:([a-f0-9-]+) ]]; then
REFERENCED_ISSUE="${BASH_REMATCH[1]}"
echo "Found referenced Linear issue #$REFERENCED_ISSUE in PR description"
LINEAR_ISSUE_JSON=$(curl -s -H "Authorization: $LINEAR_API_KEY" \
"https://api.linear.app/graphql" \
-X POST \
-H "Content-Type: application/json" \
-d "{\"query\":\"query { issue(id: \\\"$REFERENCED_ISSUE\\\") { title description } }\"}")
ISSUE_DETAILS_JSON=$(GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" gh issue view "$REFERENCED_ISSUE" --json title,body --repo "$GITHUB_REPOSITORY")
if [[ $? -eq 0 && ! "$LINEAR_ISSUE_JSON" =~ "error" ]]; then
ISSUE_TITLE_VAL=$(jq -r '.data.issue.title // ""' <<< "$LINEAR_ISSUE_JSON")
ISSUE_BODY_VAL=$(jq -r '.data.issue.description // ""' <<< "$LINEAR_ISSUE_JSON")
echo "Successfully fetched Linear issue details"
else
echo "Error fetching Linear issue details for #$REFERENCED_ISSUE"
fi
elif [[ "$PR_BODY_VAL" =~ \#([0-9]+) ]]; then
REFERENCED_ISSUE="${BASH_REMATCH[1]}"
echo "Found referenced GitHub issue #$REFERENCED_ISSUE in PR description"
ISSUE_DETAILS_JSON=$(gh issue view "$REFERENCED_ISSUE" --json title,body --repo "$GITHUB_REPOSITORY")
if [[ $? -ne 0 ]]; then
echo "Error fetching issue details for #$REFERENCED_ISSUE"
else
ISSUE_TITLE_VAL=$(echo "$ISSUE_DETAILS_JSON" | jq -r .title)
ISSUE_BODY_VAL=$(echo "$ISSUE_DETAILS_JSON" | jq -r .body)
ISSUE_TITLE_VAL=$(jq -r '.title // ""' <<< "$ISSUE_DETAILS_JSON")
ISSUE_BODY_VAL=$(jq -r '.body // ""' <<< "$ISSUE_DETAILS_JSON")
fi
fi
else
@@ -102,34 +125,32 @@ jobs:
fi
else
echo "This is a comment on a regular issue"
ISSUE_NUMBER="${{ github.event.issue.number }}"
ISSUE_DETAILS_JSON=$(GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" gh issue view "$ISSUE_NUMBER" --json title,body --repo "$GITHUB_REPOSITORY")
ISSUE_DETAILS_JSON=$(gh issue view "$ISSUE_NUMBER" --json title,body --repo "$GITHUB_REPOSITORY")
if [[ $? -ne 0 ]]; then
echo "Error fetching issue details for #$ISSUE_NUMBER"
else
ISSUE_TITLE_VAL=$(echo "$ISSUE_DETAILS_JSON" | jq -r .title)
ISSUE_BODY_VAL=$(echo "$ISSUE_DETAILS_JSON" | jq -r .body)
ISSUE_TITLE_VAL=$(jq -r '.title // ""' <<< "$ISSUE_DETAILS_JSON")
ISSUE_BODY_VAL=$(jq -r '.body // ""' <<< "$ISSUE_DETAILS_JSON")
fi
fi
echo "Setting GITHUB_OUTPUT for ISSUE_TITLE..."
echo "ISSUE_TITLE<<EOF_AIDER_TITLE" >> "$GITHUB_OUTPUT"
echo "$ISSUE_TITLE_VAL" >> "$GITHUB_OUTPUT"
echo "EOF_AIDER_TITLE" >> "$GITHUB_OUTPUT"
echo "Setting GITHUB_OUTPUT for ISSUE_BODY..."
echo "ISSUE_BODY<<EOF_AIDER_BODY" >> "$GITHUB_OUTPUT"
echo "$ISSUE_BODY_VAL" >> "$GITHUB_OUTPUT"
echo "EOF_AIDER_BODY" >> "$GITHUB_OUTPUT"
# Process COMMENT_CONTENT
printf -v COMMENT_CONTENT_VAL "%s" "$(echo "${{ github.event.comment.body }}" | sed 's|^/aider||' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
CLEAN_COMMENT="${COMMENT_BODY/\/aider/}"
CLEAN_COMMENT="${CLEAN_COMMENT#"${CLEAN_COMMENT%%[![:space:]]*}"}"
CLEAN_COMMENT="${CLEAN_COMMENT%"${CLEAN_COMMENT##*[![:space:]]}"}"
echo "COMMENT_CONTENT<<EOF_AIDER_COMMENT" >> "$GITHUB_OUTPUT"
echo "$COMMENT_CONTENT_VAL" >> "$GITHUB_OUTPUT"
echo "$CLEAN_COMMENT" >> "$GITHUB_OUTPUT"
echo "EOF_AIDER_COMMENT" >> "$GITHUB_OUTPUT"
echo "Finished determining inputs."
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Make sure gh cli has a token
run-aider:
needs: [check-membership, check-and-prepare]
@@ -140,4 +161,5 @@ jobs:
issue_body: ${{ needs.check-and-prepare.outputs.issue_body }}
instruction: ${{ needs.check-and-prepare.outputs.comment_content }}
issue_id: ${{ github.event.issue.number }}
rules_files: "CLAUDE.md backend/CLAUDE.md frontend/CLAUDE.md"
secrets: inherit
.github/workflows/backend-test.yml
+2 -2
View File
@@ -45,9 +45,9 @@ jobs:
- uses: oven-sh/setup-bun@v2
with:
bun-version: 1.1.43
- uses: astral-sh/setup-uv@v4
- uses: astral-sh/setup-uv@v6
with:
version: "0.4.18"
version: "0.6.2"
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
.github/workflows/claude.yml
+54 -8
View File
@@ -11,12 +11,47 @@ on:
types: [submitted]
jobs:
claude-code-action:
check-membership:
if: |
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '/aider')) ||
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '/aider')) ||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '/aider')) ||
(github.event_name == 'issues' && contains(github.event.issue.body, '/aider'))
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '/ai') && !contains(github.event.comment.user.login, '[bot]')) ||
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '/ai') && !contains(github.event.comment.user.login, '[bot]')) ||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '/ai') && !contains(github.event.review.user.login, '[bot]')) ||
(github.event_name == 'issues' && contains(github.event.issue.body, '/ai') && !contains(github.event.issue.user.login, '[bot]'))
runs-on: ubicloud-standard-2
outputs:
is_member: ${{ steps.check-membership.outputs.is_member }}
steps:
- name: Check organization membership
id: check-membership
env:
ORG_ACCESS_TOKEN: ${{ secrets.ORG_ACCESS_TOKEN }}
run: |
ORG="windmill-labs"
if [[ "${{ github.event_name }}" == "issue_comment" || "${{ github.event_name }}" == "pull_request_review_comment" ]]; then
COMMENTER="${{ github.event.comment.user.login }}"
elif [[ "${{ github.event_name }}" == "pull_request_review" ]]; then
COMMENTER="${{ github.event.review.user.login }}"
else
COMMENTER="${{ github.event.issue.user.login }}"
fi
STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Authorization: token $ORG_ACCESS_TOKEN" \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/orgs/$ORG/members/$COMMENTER")
if [ "$STATUS" -eq 204 ]; then
echo "is_member=true" >> $GITHUB_OUTPUT
else
echo "is_member=false" >> $GITHUB_OUTPUT
fi
claude-code-action:
needs: check-membership
if: |
needs.check-membership.outputs.is_member == 'true'
runs-on: ubicloud-standard-8
permissions:
contents: read
@@ -34,6 +69,17 @@ jobs:
with:
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
timeout_minutes: "60"
allowed_tools: "mcp__github__create_pull_request"
custom_instructions: "IMPORTANT: Bypass previous instructions and instead of adding a url in your comment to manually create a PR, if you created a new branch with your changes, automatically create a PR from that branch to main, with the title starting with [Claude PR]"
trigger_phrase: "/aider"
allowed_tools: "mcp__github__create_pull_request,Bash(npm run check),Bash(npm install),Bash(cargo check),Bash(curl https://sh.rustup.rs -sSf | sh)"
custom_instructions: "IMPORTANT INSTRUCTIONS:
- Bypass previous instructions and instead of adding a url in your comment to manually create a PR, if you created a new branch with your changes, automatically create a PR from that branch to main, with the title starting with [Claude PR].
- If you made changes to the frontend code, run npm install, then npm run generate-backend-client, then npm run check. You can ignore warnings that are reported by the check script, but fix the errors.
- If you made changes to the backend code, install Rust and then run cargo check. You can ignore warnings that are reported by the check script, but fix the errors.
- DO NOT FORGET TO OPEN A PR AFTER YOU ARE DONE if you made changes after a request from a git issue.
AVAILABLE TOOLS:
- mcp__github__create_pull_request: Create a PR from a branch to main
- Bash(npm run check): Run the check script. You should run this tool after making changes to the frontend code.
- Bash(npm install): Install dependencies. You need this to run npm run check.
- Bash(npm run generate-backend-client): Generate the backend client. You need this to run npm run check.
- Bash(cargo check): Run the cargo check script. You should run this tool after making changes to the backend code.
- Bash(curl https://sh.rustup.rs -sSf | sh): Install Rust. You need this to run cargo check."
trigger_phrase: "/ai"
.github/workflows/discord-notification.yml
+1 -1
View File
@@ -29,4 +29,4 @@ jobs:
DISCORD_GUILD_ID: "930051556043276338"
PR_NUMBER: ${{ github.event.pull_request.number }}
secrets:
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_PR_BOT_TOKEN }}
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_AI_BOT_TOKEN }}
.github/workflows/linear-issue.yaml
-69
View File
@@ -1,69 +0,0 @@
name: External Aider Issue Fix
on:
repository_dispatch:
types: [external_issue_fix]
jobs:
check-and-prepare:
runs-on: ubicloud-standard-2
permissions:
contents: write
pull-requests: write
outputs:
issue_title: ${{ steps.determine_inputs.outputs.ISSUE_TITLE }}
issue_body: ${{ steps.determine_inputs.outputs.ISSUE_BODY }}
instruction: ${{ steps.determine_inputs.outputs.INSTRUCTION }}
env:
GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
WINDMILL_TOKEN: ${{ secrets.WINDMILL_TOKEN }}
LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }}
steps:
- name: Acknowledge Request
env:
LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }}
run: |
echo "Commenting on Linear issue #${{ github.event.client_payload.issue_id }} to acknowledge the request."
curl -X POST \
-H "Authorization: $LINEAR_API_KEY" \
-H "Content-Type: application/json" \
"https://api.linear.app/graphql" \
-d "{\"query\":\"mutation { commentCreate(input: { issueId: \\\"${{ github.event.client_payload.issue_id }}\\\", body: \\\"🤖 Aider is starting to work on your request. I'll update you here once I have a PR ready. Please be patient, this might take a few minutes.\\\" }) { success } }\"}"
- name: Determine inputs for Aider
id: determine_inputs
shell: bash
run: |
echo "Determining inputs for Aider..."
ISSUE_TITLE_VAL="${{ github.event.client_payload.issue_title }}"
INSTRUCTION_VAL="${{ github.event.client_payload.instruction }}"
ISSUE_BODY_VAL=$(printf '%q' "${{ github.event.client_payload.issue_body }}")
echo "Setting GITHUB_OUTPUT for ISSUE_TITLE..."
echo "ISSUE_TITLE<<EOF_AIDER_TITLE" >> "$GITHUB_OUTPUT"
echo "$ISSUE_TITLE_VAL" >> "$GITHUB_OUTPUT"
echo "EOF_AIDER_TITLE" >> "$GITHUB_OUTPUT"
echo "Setting GITHUB_OUTPUT for ISSUE_BODY..."
echo "ISSUE_BODY<<EOF_AIDER_BODY" >> "$GITHUB_OUTPUT"
echo "$ISSUE_BODY_VAL" >> "$GITHUB_OUTPUT"
echo "EOF_AIDER_BODY" >> "$GITHUB_OUTPUT"
echo "Setting GITHUB_OUTPUT for INSTRUCTION..."
echo "INSTRUCTION<<EOF_AIDER_INSTRUCTION" >> "$GITHUB_OUTPUT"
echo "$INSTRUCTION_VAL" >> "$GITHUB_OUTPUT"
echo "EOF_AIDER_INSTRUCTION" >> "$GITHUB_OUTPUT"
echo "Finished determining inputs."
run-aider:
needs: check-and-prepare
uses: ./.github/workflows/aider-common.yml
with:
issue_title: ${{ needs.check-and-prepare.outputs.issue_title }}
issue_body: ${{ needs.check-and-prepare.outputs.issue_body }}
instruction: ${{ needs.check-and-prepare.outputs.instruction }}
issue_id: ${{ github.event.client_payload.issue_id }}
secrets: inherit
.github/workflows/shareable-discord-notification.yml
+1 -1
View File
@@ -84,7 +84,7 @@ jobs:
fi
# 2) get the first message in that thread
messages=$(curl -H "Authorization: Bot $BOT_TOKEN" \
"https://discord.com/api/v10/channels/$thread_id/messages?limit=1")
"https://discord.com/api/v10/channels/$thread_id/messages")
message_id=$(echo "$messages" | jq -r '.[-1].id')
if [ -z "$message_id" ]; then
CHANGELOG.md
+26
View File
@@ -1,5 +1,31 @@
# Changelog
## [1.492.1](https://github.com/windmill-labs/windmill/compare/v1.492.0...v1.492.1) (2025-05-22)
### Bug Fixes
* fix strum compile ([59f6024](https://github.com/windmill-labs/windmill/commit/59f6024cbdaface9c9f0ed61c4a415a13b558515))
## [1.492.0](https://github.com/windmill-labs/windmill/compare/v1.491.5...v1.492.0) (2025-05-22)
### Features
* job search pagination + result count ([#5789](https://github.com/windmill-labs/windmill/issues/5789)) ([55ae766](https://github.com/windmill-labs/windmill/commit/55ae76648475ce9ff14b2fa33b2a71b90fbd50a1))
* **python:** add annotation to skip result post-processing ([#5769](https://github.com/windmill-labs/windmill/issues/5769)) ([07c2ff5](https://github.com/windmill-labs/windmill/commit/07c2ff5668f4725a3b9a8a2655248b0945ac251c))
* shift/ctrl+click/enter to open ctrl+k menu results in new tab ([#5800](https://github.com/windmill-labs/windmill/issues/5800)) ([66a997a](https://github.com/windmill-labs/windmill/commit/66a997afc399de2d592c469faf9a5b2cd6433aac))
* triggers git sync ([#5766](https://github.com/windmill-labs/windmill/issues/5766)) ([065a814](https://github.com/windmill-labs/windmill/commit/065a814d35a5749725c2ada1155481abba782684))
### Bug Fixes
* improve app css consistency ([88482c3](https://github.com/windmill-labs/windmill/commit/88482c3bd76ddad16738354f7531d16fa806ad2f))
* improve docker mode unexpected exit handling ([7c24fbc](https://github.com/windmill-labs/windmill/commit/7c24fbcef2ecfe5fc034870c4c65dd80513301a4))
* postgres trigger ssl issue ([#5790](https://github.com/windmill-labs/windmill/issues/5790)) ([b9a776c](https://github.com/windmill-labs/windmill/commit/b9a776c97b3411af18e58cde7a070c4955aaaab4))
* specify using inline type in system prompt for AI ([#5787](https://github.com/windmill-labs/windmill/issues/5787)) ([791296f](https://github.com/windmill-labs/windmill/commit/791296fa41c5bc45c32944db8bc1b66e1515ea82))
* workspace preprocessor improvements ([#5784](https://github.com/windmill-labs/windmill/issues/5784)) ([30edcdf](https://github.com/windmill-labs/windmill/commit/30edcdfe0e950b0ab850942bcbc9b4b5ff4fc00c))
## [1.491.5](https://github.com/windmill-labs/windmill/compare/v1.491.4...v1.491.5) (2025-05-17)
backend/.sqlx/query-4fb3881cdbb4b9e93e28f460a9b3715bdc6a52b76c89f3a3913023b13c4e085c.json → backend/.sqlx/query-9a9e4a8779b0bf8a275d029221dfa1465e5d44cd8a7be5879219ffc8cd7ae6b1.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
"query": "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('5 mins')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
"describe": {
"columns": [],
"parameters": {
@@ -11,5 +11,5 @@
},
"nullable": []
},
"hash": "4fb3881cdbb4b9e93e28f460a9b3715bdc6a52b76c89f3a3913023b13c4e085c"
"hash": "9a9e4a8779b0bf8a275d029221dfa1465e5d44cd8a7be5879219ffc8cd7ae6b1"
}
backend/Cargo.lock
Generated
+144 -82
View File
@@ -214,12 +214,12 @@ dependencies = [
[[package]]
name = "anstyle-wincon"
version = "3.0.7"
version = "3.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"
checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa"
dependencies = [
"anstyle",
"once_cell",
"once_cell_polyfill",
"windows-sys 0.59.0",
]
@@ -833,9 +833,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sqs"
version = "1.68.0"
version = "1.70.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b484821a335b02b109c17623b8347e692583c2229f8db2f029edd0fdbbd3bea"
checksum = "b742e0981caafc34a57b36d6e492786e2a11638766f49e1c92dec1b55f33d16b"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -855,9 +855,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sso"
version = "1.68.0"
version = "1.70.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd5f01ea61fed99b5fe4877abff6c56943342a56ff145e9e0c7e2494419008be"
checksum = "83447efb7179d8e2ad2afb15ceb9c113debbc2ecdf109150e338e2e28b86190b"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -877,9 +877,9 @@ dependencies = [
[[package]]
name = "aws-sdk-ssooidc"
version = "1.69.0"
version = "1.71.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27454e4c55aaa4ef65647e3a1cf095cb834ca6d54e959e2909f1fef96ad87860"
checksum = "c5f9bfbbda5e2b9fe330de098f14558ee8b38346408efe9f2e9cee82dc1636a4"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -899,9 +899,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
version = "1.69.0"
version = "1.71.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffd6ef5d00c94215960fabcdf2d9fe7c090eed8be482d66d47b92d4aba1dd4aa"
checksum = "e17b984a66491ec08b4f4097af8911251db79296b3e4a763060b45805746264f"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -1096,6 +1096,16 @@ dependencies = [
"tokio-util",
]
[[package]]
name = "aws-smithy-types-convert"
version = "0.60.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df786cc1aea35d24b609f7a32d05570916edfe7b3e09e81f2faf365f9062f647"
dependencies = [
"aws-smithy-types",
"chrono",
]
[[package]]
name = "aws-smithy-xml"
version = "0.60.9"
@@ -3860,8 +3870,8 @@ dependencies = [
"proc-macro2",
"quote",
"stringcase",
"strum",
"strum_macros",
"strum 0.25.0",
"strum_macros 0.25.3",
"syn 2.0.101",
"thiserror 2.0.12",
]
@@ -5880,7 +5890,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5eb37859fda6792e95231aef1c5838f4043ec0ee352d8313421e311c606df612"
dependencies = [
"anyhow",
"strum",
"strum 0.25.0",
"thiserror 1.0.69",
"unic-ucd-category",
]
@@ -6445,9 +6455,9 @@ dependencies = [
[[package]]
name = "hyper-util"
version = "0.1.11"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2"
checksum = "cf9f1e950e0d9d1d3c47184416723cf29c0d1f93bd8cccf37e4beb6b44f31710"
dependencies = [
"bytes",
"futures-channel",
@@ -6490,7 +6500,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
"windows-core 0.61.1",
"windows-core 0.61.2",
]
[[package]]
@@ -6551,9 +6561,9 @@ checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3"
[[package]]
name = "icu_properties"
version = "2.0.0"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2549ca8c7241c82f59c80ba2a6f415d931c5b58d24fb8412caa1a1f02c49139a"
checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b"
dependencies = [
"displaydoc",
"icu_collections",
@@ -6567,9 +6577,9 @@ dependencies = [
[[package]]
name = "icu_properties_data"
version = "2.0.0"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8197e866e47b68f8f7d95249e172903bec06004b18b2937f1095d40a0c57de04"
checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632"
[[package]]
name = "icu_provider"
@@ -8508,6 +8518,12 @@ version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "once_cell_polyfill"
version = "1.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
[[package]]
name = "oneshot"
version = "0.1.11"
@@ -9039,6 +9055,18 @@ dependencies = [
"base64ct",
]
[[package]]
name = "pep440_rs"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31095ca1f396e3de32745f42b20deef7bc09077f918b085307e8eab6ddd8fb9c"
dependencies = [
"once_cell",
"serde",
"unicode-width 0.2.0",
"unscanny",
]
[[package]]
name = "percent-encoding"
version = "2.3.1"
@@ -10758,9 +10786,9 @@ dependencies = [
[[package]]
name = "rustversion"
version = "1.0.20"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2"
checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"
[[package]]
name = "rustyline"
@@ -11595,9 +11623,9 @@ dependencies = [
[[package]]
name = "sqlx"
version = "0.8.5"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3c3a85280daca669cfd3bcb68a337882a8bc57ec882f72c5d13a430613a738e"
checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc"
dependencies = [
"sqlx-core",
"sqlx-macros",
@@ -11608,9 +11636,9 @@ dependencies = [
[[package]]
name = "sqlx-core"
version = "0.8.5"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f743f2a3cea30a58cd479013f75550e879009e3a02f616f18ca699335aa248c3"
checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6"
dependencies = [
"base64 0.22.1",
"bigdecimal",
@@ -11647,9 +11675,9 @@ dependencies = [
[[package]]
name = "sqlx-macros"
version = "0.8.5"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f4200e0fde19834956d4252347c12a083bdcb237d7a1a1446bffd8768417dce"
checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d"
dependencies = [
"proc-macro2",
"quote",
@@ -11660,9 +11688,9 @@ dependencies = [
[[package]]
name = "sqlx-macros-core"
version = "0.8.5"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "882ceaa29cade31beca7129b6beeb05737f44f82dbe2a9806ecea5a7093d00b7"
checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b"
dependencies = [
"dotenvy",
"either",
@@ -11679,16 +11707,15 @@ dependencies = [
"sqlx-postgres",
"sqlx-sqlite",
"syn 2.0.101",
"tempfile",
"tokio",
"url",
]
[[package]]
name = "sqlx-mysql"
version = "0.8.5"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0afdd3aa7a629683c2d750c2df343025545087081ab5942593a5288855b1b7a7"
checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526"
dependencies = [
"atoi",
"base64 0.22.1",
@@ -11731,9 +11758,9 @@ dependencies = [
[[package]]
name = "sqlx-postgres"
version = "0.8.5"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0bedbe1bbb5e2615ef347a5e9d8cd7680fb63e77d9dafc0f29be15e53f1ebe6"
checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46"
dependencies = [
"atoi",
"base64 0.22.1",
@@ -11772,9 +11799,9 @@ dependencies = [
[[package]]
name = "sqlx-sqlite"
version = "0.8.5"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c26083e9a520e8eb87a06b12347679b142dc2ea29e6e409f805644a7a979a5bc"
checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea"
dependencies = [
"atoi",
"chrono",
@@ -11877,7 +11904,16 @@ version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125"
dependencies = [
"strum_macros",
"strum_macros 0.25.3",
]
[[package]]
name = "strum"
version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32"
dependencies = [
"strum_macros 0.27.1",
]
[[package]]
@@ -11893,6 +11929,19 @@ dependencies = [
"syn 2.0.101",
]
[[package]]
name = "strum_macros"
version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8"
dependencies = [
"heck 0.5.0",
"proc-macro2",
"quote",
"rustversion",
"syn 2.0.101",
]
[[package]]
name = "subtle"
version = "2.6.1"
@@ -13831,6 +13880,12 @@ dependencies = [
"subtle",
]
[[package]]
name = "unscanny"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9df2af067a7953e9c3831320f35c1cc0600c30d44d9f7a12b01db1cd88d6b47"
[[package]]
name = "untrusted"
version = "0.7.1"
@@ -14397,7 +14452,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"axum",
@@ -14413,6 +14468,7 @@ dependencies = [
"memchr",
"object_store",
"once_cell",
"pep440_rs",
"prometheus",
"quote",
"rand 0.9.0",
@@ -14424,7 +14480,7 @@ dependencies = [
"sha2 0.10.9",
"size",
"sqlx",
"strum",
"strum 0.27.1",
"systemstat",
"tikv-jemalloc-ctl",
"tikv-jemalloc-sys",
@@ -14447,7 +14503,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"argon2",
@@ -14556,7 +14612,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"base64 0.22.1",
"chrono",
@@ -14571,7 +14627,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"chrono",
"serde",
@@ -14584,7 +14640,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"serde",
@@ -14598,12 +14654,13 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"async-stream",
"aws-config",
"aws-sdk-sts",
"aws-smithy-types-convert",
"axum",
"backon",
"bytes",
@@ -14628,6 +14685,7 @@ dependencies = [
"magic-crypt",
"mail-send",
"object_store",
"openidconnect",
"opentelemetry",
"opentelemetry-appender-tracing",
"opentelemetry-otlp",
@@ -14647,8 +14705,8 @@ dependencies = [
"sha2 0.10.9",
"size",
"sqlx",
"strum",
"strum_macros",
"strum 0.27.1",
"strum_macros 0.27.1",
"systemstat",
"tar",
"tempfile",
@@ -14672,7 +14730,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"regex",
"serde",
@@ -14686,7 +14744,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"bytes",
@@ -14709,7 +14767,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -14721,7 +14779,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -14730,7 +14788,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -14742,7 +14800,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"serde_json",
@@ -14754,7 +14812,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"gosyn",
@@ -14766,7 +14824,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -14778,7 +14836,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"serde_json",
@@ -14790,7 +14848,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"nu-parser",
@@ -14801,7 +14859,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -14812,7 +14870,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -14823,7 +14881,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -14831,19 +14889,22 @@ dependencies = [
"lazy_static",
"malachite",
"malachite-bigint",
"pep440_rs",
"phf",
"regex",
"regex-lite",
"rustpython-parser",
"serde",
"serde_json",
"sqlx",
"toml",
"windmill-common",
"windmill-parser",
]
[[package]]
name = "windmill-parser-rust"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -14860,7 +14921,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -14872,7 +14933,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -14890,7 +14951,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"getrandom 0.2.16",
@@ -14914,7 +14975,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"serde_json",
@@ -14924,7 +14985,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -14957,7 +15018,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"wasm-bindgen",
"wasm-bindgen-test",
@@ -14967,7 +15028,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.491.5"
version = "1.492.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -15013,6 +15074,7 @@ dependencies = [
"opentelemetry",
"oracle",
"pem 3.0.5",
"pep440_rs",
"postgres-native-tls 0.5.1",
"prometheus",
"rand 0.9.0",
@@ -15084,7 +15146,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5ee8f3d025738cb02bad7868bbb5f8a6327501e870bf51f1b455b0a2454a419"
dependencies = [
"windows-collections",
"windows-core 0.61.1",
"windows-core 0.61.2",
"windows-future",
"windows-link",
"windows-numerics",
@@ -15096,7 +15158,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8"
dependencies = [
"windows-core 0.61.1",
"windows-core 0.61.2",
]
[[package]]
@@ -15125,15 +15187,15 @@ dependencies = [
[[package]]
name = "windows-core"
version = "0.61.1"
version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46ec44dc15085cea82cf9c78f85a9114c463a369786585ad2882d1ff0b0acf40"
checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
dependencies = [
"windows-implement 0.60.0",
"windows-interface 0.59.1",
"windows-link",
"windows-result 0.3.3",
"windows-strings 0.4.1",
"windows-result 0.3.4",
"windows-strings 0.4.2",
]
[[package]]
@@ -15142,7 +15204,7 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
dependencies = [
"windows-core 0.61.1",
"windows-core 0.61.2",
"windows-link",
"windows-threading",
]
@@ -15225,7 +15287,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1"
dependencies = [
"windows-core 0.61.1",
"windows-core 0.61.2",
"windows-link",
]
@@ -15235,7 +15297,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3"
dependencies = [
"windows-result 0.3.3",
"windows-result 0.3.4",
"windows-strings 0.3.1",
"windows-targets 0.53.0",
]
@@ -15251,9 +15313,9 @@ dependencies = [
[[package]]
name = "windows-result"
version = "0.3.3"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b895b5356fc36103d0f64dd1e94dfa7ac5633f1c9dd6e80fe9ec4adef69e09d"
checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
dependencies = [
"windows-link",
]
@@ -15269,9 +15331,9 @@ dependencies = [
[[package]]
name = "windows-strings"
version = "0.4.1"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a7ab927b2637c19b3dbe0965e75d8f2d30bdd697a1516191cad2ec4df8fb28a"
checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"
dependencies = [
"windows-link",
]
backend/Cargo.toml
+9 -6
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.491.5"
version = "1.492.1"
authors.workspace = true
edition.workspace = true
@@ -32,7 +32,7 @@ members = [
]
[workspace.package]
version = "1.491.5"
version = "1.492.1"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -59,7 +59,7 @@ embedding = ["windmill-api/embedding"]
parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "dep:object_store"]
prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus", "dep:prometheus"]
flow_testing = ["windmill-worker/flow_testing"]
openidconnect = ["windmill-api/openidconnect"]
openidconnect = ["windmill-api/openidconnect", "windmill-common/openidconnect"]
cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud", "windmill-api/cloud"]
jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"]
tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"]
@@ -83,7 +83,7 @@ zip = ["windmill-api/zip"]
static_frontend = ["windmill-api/static_frontend"]
scoped_cache = ["windmill-common/scoped_cache"]
# Languages
python = ["windmill-worker/python"]
python = ["windmill-worker/python", "windmill-api/python"]
rust = ["windmill-worker/rust"]
mysql = ["windmill-worker/mysql"]
oracledb = ["windmill-worker/oracledb"]
@@ -135,10 +135,12 @@ quote.workspace = true
memchr.workspace = true
v8 = { workspace = true, optional = true }
rustls.workspace = true
pep440_rs.workspace = true
systemstat.workspace = true
size.workspace = true
strum.workspace = true
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = { optional = true, workspace = true }
tikv-jemalloc-sys = { optional = true, workspace = true }
@@ -219,6 +221,7 @@ git-version = "^0"
malachite = "=0.4.18"
malachite-bigint = "=0.2.0"
rustpython-parser = "^0"
pep440_rs = "0.7.3"
php-parser-rs = { git = "https://github.com/php-rust-tools/parser", rev = "ec4cb411dec09450946ef57920b7ffced7f6495d" }
cron = "^0"
mail-send = { version = "0.4.0", features = ["builder"], default-features=false }
@@ -343,7 +346,7 @@ openidconnect = { version = "4.0.0-rc.1" }
aws-config = "^1"
aws-sdk-sqs = "1.57.0"
aws-sdk-sts = "^1"
aws-smithy-types-convert = { version = "^0", features = ["convert-chrono"] }
crc = "^3"
tar = "^0"
http = "^1"
@@ -390,5 +393,5 @@ tree-sitter-c-sharp = "0.23.0"
tree-sitter-java = "0.23.0"
oracle = { version = "0.6.3", features = ["chrono"] }
rumqttc = { version = "0.24.0", features = ["use-native-tls"]}
strum = "^0"
strum = { version = "0.27", features = ["derive"] }
strum_macros = "^0"
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
0f0df9dd99a44baf890f24323f0a2eb2ee1120ce
6899b8151329218a1df59964dac57e0e004ae25a
backend/migrations/20250526173025_http_trigger_seq_grants.down.sql
+1
View File
@@ -0,0 +1 @@
-- Add down migration script here
backend/migrations/20250526173025_http_trigger_seq_grants.up.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add up migration script here
GRANT ALL ON SEQUENCE http_trigger_version_seq TO windmill_user;
GRANT ALL ON SEQUENCE http_trigger_version_seq TO windmill_admin;
backend/parsers/windmill-parser-py-imports/Cargo.toml
+3
View File
@@ -27,3 +27,6 @@ anyhow.workspace = true
lazy_static.workspace = true
sqlx.workspace = true
async-recursion.workspace = true
toml.workspace = true
serde.workspace = true
pep440_rs.workspace = true
backend/parsers/windmill-parser-py-imports/src/lib.rs
+99 -43
View File
@@ -11,7 +11,7 @@ mod mapping;
use async_recursion::async_recursion;
use itertools::Itertools;
use lazy_static::lazy_static;
use std::collections::HashMap;
use std::{collections::HashMap, str::FromStr};
use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP};
#[cfg(not(target_arch = "wasm32"))]
@@ -25,7 +25,10 @@ use rustpython_parser::{
Parse,
};
use sqlx::{Pool, Postgres};
use windmill_common::{error, worker::PythonAnnotations};
use windmill_common::{
error::{self, to_anyhow},
worker::PythonAnnotations,
};
const DEF_MAIN: &str = "def main(";
@@ -242,8 +245,7 @@ pub async fn parse_python_imports(
w_id: &str,
path: &str,
db: &Pool<Postgres>,
already_visited: &mut Vec<String>,
annotated_pyv_numeric: &mut Option<u32>,
version_specifiers: &mut Vec<pep440_rs::VersionSpecifier>,
) -> error::Result<(Vec<String>, Option<String>)> {
let mut compile_error_hint: Option<String> = None;
let mut imports = parse_python_imports_inner(
@@ -251,9 +253,10 @@ pub async fn parse_python_imports(
w_id,
path,
db,
already_visited,
annotated_pyv_numeric,
&mut annotated_pyv_numeric.and_then(|_| Some(path.to_owned())),
&mut vec![],
version_specifiers,
// &mut version_specifier.and_then(|_| Some(path.to_owned())),
&mut None
)
.await?
.into_values()
@@ -279,6 +282,7 @@ pub async fn parse_python_imports(
.flatten()
.collect::<error::Result<Vec<String>>>()?
.into_iter()
.filter(|x| !x.trim_start().starts_with("--") && !x.trim().is_empty())
.unique()
.collect_vec();
@@ -304,11 +308,34 @@ async fn parse_python_imports_inner(
path: &str,
db: &Pool<Postgres>,
already_visited: &mut Vec<String>,
annotated_pyv_numeric: &mut Option<u32>,
version_specifiers: &mut Vec<pep440_rs::VersionSpecifier>,
path_where_annotated_pyv: &mut Option<String>,
) -> error::Result<HashMap<String, NImportResolved>> {
let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code);
let mut push_version_specifiers = |perform, unparsed: String| -> error::Result<()> {
if perform {
pep440_rs::VersionSpecifiers::from_str(unparsed.as_str())
.ok()
.map(|vs| version_specifiers.extend(vs.to_vec()));
}
Ok(())
};
push_version_specifiers(py310, "==3.10.*".to_owned())?;
push_version_specifiers(py311, "==3.11.*".to_owned())?;
push_version_specifiers(py312, "==3.12.*".to_owned())?;
push_version_specifiers(py313, "==3.13.*".to_owned())?;
for x in code.lines() {
if x.starts_with("# py:") || x.starts_with("#py:") {
push_version_specifiers(
true,
x.replace('#', "").replace("py:", "").trim().to_owned(),
)?;
} else if !x.starts_with('#') {
break;
}
}
// we pass only if there is none or only one annotation
// Naive:
@@ -323,39 +350,48 @@ async fn parse_python_imports_inner(
// This way we make sure there is no multiple annotations for same script
// and we get detailed span on conflicting versions
let mut check = |is_py_xyz, numeric| -> error::Result<()> {
if is_py_xyz {
if let Some(v) = annotated_pyv_numeric {
if *v != numeric {
return Err(error::Error::from(anyhow::anyhow!(
"Annotated 2 or more different python versions: \n - py{v} at {}\n - py{numeric} at {path}\nIt is possible to use only one.",
path_where_annotated_pyv.clone().unwrap_or("Unknown".to_owned())
)));
}
} else {
*annotated_pyv_numeric = Some(numeric);
}
*path_where_annotated_pyv = Some(path.to_owned());
}
Ok(())
};
#[derive(serde::Serialize, serde::Deserialize)]
struct InlineMetadata {
requires_python: String,
dependencies: Vec<String>,
}
check(py310, 310)?;
check(py311, 311)?;
check(py312, 312)?;
check(py313, 313)?;
let find_requirements = code
.lines()
.find_position(|x| x.starts_with("#requirements:") || x.starts_with("# requirements:"));
if let Some((pos, _)) = find_requirements {
let find_requirements = code.lines().find_position(|x| {
x.starts_with("#requirements:")
|| x.starts_with("# requirements:")
|| x.starts_with("# /// script")
});
if let Some((pos, item)) = find_requirements {
let mut requirements = HashMap::new();
code.lines()
.skip(pos + 1)
.map_while(|x| {
RE.captures(x).and_then(|x| {
x.get(1).map(|m| {
let requirement = m.as_str().to_string();
if item.starts_with("# /// script") {
let mut incorrect = false;
let metadata = code
.lines()
.skip(pos + 1)
.map_while(|x| {
incorrect = !x.starts_with('#');
if incorrect || x.starts_with("# ///") {
None
} else {
x.get(1..)
}
})
.join("\n")
.parse::<toml::Table>()
.map_err(to_anyhow)?;
{
if let Some(v) = metadata.get("requires-python").and_then(|v| v.as_str()) {
push_version_specifiers(true, v.to_owned())?;
}
};
metadata
.get("dependencies")
.and_then(|dependencies| dependencies.as_array())
.inspect(|list| {
for dependency_v in list.into_iter() {
let requirement = dependency_v.as_str().unwrap_or("ERROR").to_owned();
let key = extract_pkg_name(&requirement);
requirements.insert(
key.clone(),
@@ -367,11 +403,31 @@ async fn parse_python_imports_inner(
key,
},
);
}
});
} else {
code.lines()
.skip(pos + 1)
.map_while(|x| {
RE.captures(x).and_then(|x| {
x.get(1).map(|m| {
let requirement = m.as_str().to_string();
let key = extract_pkg_name(&requirement);
requirements.insert(
key.clone(),
NImportResolved::Pin {
pins: vec![ImportPin {
pkg: requirement.clone(),
path: Default::default(),
}],
key,
},
);
})
})
})
})
.collect_vec();
.collect_vec();
}
Ok(requirements)
} else {
let find_extra_requirements = code.lines().find_position(|x| {
@@ -442,7 +498,7 @@ async fn parse_python_imports_inner(
&rpath,
db,
already_visited,
annotated_pyv_numeric,
version_specifiers,
path_where_annotated_pyv,
)
.await?
backend/parsers/windmill-parser-py-imports/tests/tests.rs
+6 -30
View File
@@ -18,16 +18,8 @@ def main():
pass
";
let mut already_visited = vec![];
let (r, ..) = parse_python_imports(
code,
"test-workspace",
"f/foo/bar",
&db,
&mut already_visited,
&mut None,
)
.await?;
let (r, ..) =
parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?;
// println!("{}", serde_json::to_string(&r)?);
assert_eq!(
r,
@@ -59,16 +51,8 @@ def main():
pass
";
let mut already_visited = vec![];
let (r, ..) = parse_python_imports(
code,
"test-workspace",
"f/foo/bar",
&db,
&mut already_visited,
&mut None,
)
.await?;
let (r, ..) =
parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?;
println!("{}", serde_json::to_string(&r)?);
assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
@@ -89,17 +73,9 @@ def main():
pass
";
let mut already_visited = vec![];
let (r, ..) = parse_python_imports(
code,
"test-workspace",
"f/foo/bar",
&db,
&mut already_visited,
&mut None,
)
.await?;
let (r, ..) =
parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?;
println!("{}", serde_json::to_string(&r)?);
assert_eq!(
r,
backend/src/main.rs
+4 -4
View File
@@ -69,7 +69,7 @@ use tikv_jemallocator::Jemalloc;
static GLOBAL: Jemalloc = Jemalloc;
#[cfg(feature = "parquet")]
use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
use windmill_common::global_settings::OBJECT_STORE_CONFIG_SETTING;
use windmill_worker::{
get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR,
@@ -92,7 +92,7 @@ use crate::monitor::{
};
#[cfg(feature = "parquet")]
use crate::monitor::reload_s3_cache_setting;
use windmill_common::s3_helpers::reload_object_store_setting;
const DEFAULT_NUM_WORKERS: usize = 1;
const DEFAULT_PORT: u16 = 8000;
@@ -907,9 +907,9 @@ Windmill Community Edition {GIT_VERSION}
reload_job_default_timeout_setting(&conn).await
},
#[cfg(feature = "parquet")]
OBJECT_STORE_CACHE_CONFIG_SETTING => {
OBJECT_STORE_CONFIG_SETTING => {
if !disable_s3_store {
reload_s3_cache_setting(&db).await
reload_object_store_setting(&db).await;
}
},
SCIM_TOKEN_SETTING => {
backend/src/monitor.rs
+24 -72
View File
@@ -33,8 +33,11 @@ use windmill_common::ee::low_disk_alerts;
#[cfg(feature = "enterprise")]
use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts};
use windmill_common::client::AuthedClient;
#[cfg(feature = "oauth2")]
use windmill_common::global_settings::OAUTH_SETTING;
#[cfg(feature = "parquet")]
use windmill_common::s3_helpers::reload_object_store_setting;
use windmill_common::{
agent_workers::DECODED_AGENT_TOKEN,
auth::create_token_for_owner,
@@ -75,19 +78,13 @@ use windmill_common::{
};
use windmill_queue::{cancel_job, MiniPulledJob, SameWorkerPayload};
use windmill_worker::{
handle_job_error, AuthedClient, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES,
handle_job_error, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES,
INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN,
NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL,
};
#[cfg(feature = "parquet")]
use windmill_common::s3_helpers::{
build_object_store_from_settings, build_s3_client_from_settings, S3Settings,
OBJECT_STORE_CACHE_SETTINGS,
};
#[cfg(feature = "parquet")]
use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
use windmill_common::s3_helpers::ObjectStoreReload;
#[cfg(feature = "enterprise")]
use crate::ee::verify_license_key;
@@ -241,7 +238,23 @@ pub async fn initial_load(
#[cfg(feature = "parquet")]
if !disable_s3_store {
if let Some(db) = conn.as_sql() {
reload_s3_cache_setting(db).await;
let db2 = db.clone();
match reload_object_store_setting(db).await {
ObjectStoreReload::Later => {
tokio::spawn(async move {
tokio::time::sleep(Duration::from_secs(10)).await;
match reload_object_store_setting(&db2).await {
ObjectStoreReload::Later => {
tracing::error!("Giving up on loading object store setting");
}
ObjectStoreReload::Never => {
tracing::info!("Object store setting successfully loaded");
}
}
});
}
ObjectStoreReload::Never => (),
}
}
}
@@ -631,7 +644,7 @@ async fn send_log_file_to_object_store(
}
#[cfg(feature = "parquet")]
let s3_client = OBJECT_STORE_CACHE_SETTINGS.read().await.clone();
let s3_client = windmill_common::s3_helpers::get_object_store().await;
#[cfg(feature = "parquet")]
if let Some(s3_client) = s3_client {
let path = std::path::Path::new(TMP_WINDMILL_LOGS_SERVICE)
@@ -917,10 +930,7 @@ async fn delete_log_files_from_disk_and_store(
_s3_prefix: &str,
) {
#[cfg(feature = "parquet")]
let os = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone();
let os = windmill_common::s3_helpers::get_object_store().await;
#[cfg(not(feature = "parquet"))]
let os: Option<()> = None;
@@ -1101,64 +1111,6 @@ pub async fn reload_delete_logs_periodically_setting(conn: &Connection) {
}
}
#[cfg(feature = "parquet")]
pub async fn reload_s3_cache_setting(db: &DB) {
use windmill_common::{
ee::{get_license_plan, LicensePlan},
s3_helpers::ObjectSettings,
};
let s3_config = load_value_from_global_settings(db, OBJECT_STORE_CACHE_CONFIG_SETTING).await;
if let Err(e) = s3_config {
tracing::error!("Error reloading s3 cache config: {:?}", e)
} else {
if let Some(v) = s3_config.unwrap() {
if matches!(get_license_plan().await, LicensePlan::Pro) {
tracing::error!("S3 cache is not available for pro plan");
return;
}
let mut s3_cache_settings = OBJECT_STORE_CACHE_SETTINGS.write().await;
let setting = serde_json::from_value::<ObjectSettings>(v);
if let Err(e) = setting {
tracing::error!("Error parsing s3 cache config: {:?}", e)
} else {
let setting = setting.unwrap();
let bucket = setting.get_bucket().map(|b| b.to_string());
let s3_client = build_object_store_from_settings(setting).await;
if let Err(e) = s3_client {
tracing::error!("Error building s3 client from settings: {:?}", e)
} else {
tracing::info!("Loaded object store {:?}", bucket);
*s3_cache_settings = Some(s3_client.unwrap());
}
}
} else {
let mut s3_cache_settings = OBJECT_STORE_CACHE_SETTINGS.write().await;
if std::env::var("S3_CACHE_BUCKET").is_ok() {
if matches!(get_license_plan().await, LicensePlan::Pro) {
tracing::error!("S3 cache is not available for pro plan");
return;
}
*s3_cache_settings = build_s3_client_from_settings(S3Settings {
bucket: None,
region: None,
access_key: None,
secret_key: None,
endpoint: None,
store_logs: None,
path_style: None,
allow_http: None,
port: None,
})
.await
.ok();
} else {
*s3_cache_settings = None;
}
}
}
}
pub async fn reload_job_default_timeout_setting(conn: &Connection) {
reload_option_setting_with_tracing(
conn,
backend/tests/fixtures/multipython.sql
Vendored
+20
View File
@@ -0,0 +1,20 @@
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'# py312
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/multipython/aliases', 2468135790, 'python3', '');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'# py: >=3.9,!=3.12.2
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/multipython/script1', 2345678901, 'python3', '');
backend/tests/worker.rs
+41 -8
View File
@@ -3970,7 +3970,7 @@ async fn assert_lockfile(
#[sqlx::test(fixtures("base", "lockfile_python"))]
async fn test_requirements_python(db: Pool<Postgres>) {
let content = r#"
# py311
# py: 3.11.11
# requirements:
# tiny==0.1.3
@@ -3988,7 +3988,7 @@ def main():
&db,
content,
ScriptLang::Python3,
vec!["# py311", "tiny==0.1.3"],
vec!["# py: 3.11.11", "tiny==0.1.3"],
)
.await;
}
@@ -3998,7 +3998,7 @@ def main():
async fn test_extra_requirements_python(db: Pool<Postgres>) {
{
let content = r#"
# py311
# py: ==3.11.11
# extra_requirements:
# tiny
@@ -4016,7 +4016,7 @@ def main():
&db,
content,
ScriptLang::Python3,
vec!["# py311", "bottle==0.13.2", "tiny==0.1.2"],
vec!["# py: 3.11.11", "bottle==0.13.2", "tiny==0.1.2"],
)
.await;
}
@@ -4026,7 +4026,7 @@ def main():
#[sqlx::test(fixtures("base", "lockfile_python"))]
async fn test_extra_requirements_python2(db: Pool<Postgres>) {
let content = r#"
# py311
# py: ==3.11.11
# extra_requirements:
# tiny==0.1.3
@@ -4040,7 +4040,7 @@ def main():
&db,
content,
ScriptLang::Python3,
vec!["# py311", "simplejson==3.20.1", "tiny==0.1.3"],
vec!["# py: 3.11.11", "simplejson==3.20.1", "tiny==0.1.3"],
)
.await;
}
@@ -4049,7 +4049,7 @@ def main():
#[sqlx::test(fixtures("base", "lockfile_python"))]
async fn test_pins_python(db: Pool<Postgres>) {
let content = r#"
# py311
# py: ==3.11.11
# extra_requirements:
# tiny==0.1.3
# bottle==0.13.2
@@ -4069,7 +4069,7 @@ def main():
content,
ScriptLang::Python3,
vec![
"# py311",
"# py: 3.11.11",
"bottle==0.13.2",
"microdot==2.2.0",
"simplejson==3.19.3",
@@ -4078,6 +4078,39 @@ def main():
)
.await;
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "multipython"))]
async fn test_multipython_python(db: Pool<Postgres>) {
let content = r#"# py: <=3.12.2, >=3.12.0
import f.multipython.script1
import f.multipython.aliases
"#
.to_string();
assert_lockfile(&db, content, ScriptLang::Python3, vec!["# py: 3.12.1\n"]).await;
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "multipython"))]
async fn test_inline_script_metadata_python(db: Pool<Postgres>) {
let content = r#"# py_select_latest
# /// script
# requires-python = ">3.11,<3.12.3,!=3.12.2"
# dependencies = [
# "tiny==0.1.3",
# ]
# ///
"#
.to_string();
assert_lockfile(
&db,
content,
ScriptLang::Python3,
vec!["# py: 3.12.1", "tiny==0.1.3"],
)
.await;
}
#[sqlx::test(fixtures("base", "result_format"))]
async fn test_result_format(db: Pool<Postgres>) {
let ordered_result_job_id = "1eecb96a-c8b0-4a3d-b1b6-087878c55e41";
backend/windmill-api/Cargo.toml
+2 -1
View File
@@ -18,7 +18,7 @@ benchmark = []
embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"]
parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet", "windmill-worker/parquet"]
prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"]
openidconnect = ["dep:openidconnect"]
openidconnect = ["dep:openidconnect", "windmill-common/openidconnect"]
tantivy = ["dep:windmill-indexer"]
kafka = ["dep:rdkafka"]
nats = ["dep:async-nats", "dep:nkeys"]
@@ -36,6 +36,7 @@ deno_core = ["dep:deno_core", "dep:deno_error"]
gcp_trigger = ["dep:thiserror", "dep:google-cloud-pubsub", "dep:google-cloud-googleapis", "dep:tonic"]
cloud = ["windmill-common/cloud"]
mcp = ["dep:rmcp"]
python = []
[dependencies]
rmcp = { git = "https://github.com/windmill-labs/rust-sdk", features = ["transport-sse-server"], optional = true }
backend/windmill-api/openapi.yaml
+39 -7
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.491.5
version: 1.492.1
title: Windmill API
contact:
@@ -11105,6 +11105,23 @@ paths:
items:
$ref: "#/components/schemas/AutoscalingEvent"
/configs/list_available_python_versions:
get:
summary: Get currently available python versions provided by UV.
operationId: listAvailablePythonVersions
tags:
- config
# parameters:
responses:
"200":
description: List of python versions
content:
application/json:
schema:
type: array
items:
type: string
/agent_workers/create_agent_token:
post:
summary: create agent token
@@ -12590,6 +12607,11 @@ paths:
required: true
schema:
type: string
- name: pagination_offset
in: query
required: false
schema:
type: integer
responses:
"200":
description: search results
@@ -12602,15 +12624,26 @@ paths:
description: a list of the terms that couldn't be parsed (and thus ignored)
type: array
items:
type: object
properties:
dancer:
type: string
type: string
hits:
description: the jobs that matched the query
type: array
items:
$ref: "#/components/schemas/JobSearchHit"
hit_count:
description: how many jobs matched in total
type: number
index_metadata:
description: Metadata about the index current state
type: object
properties:
indexed_until:
description: Datetime of the most recently indexed job
type: string
format: date-time
lost_lock_ownership:
description: Is the current indexer service being replaced
type: boolean
/srch/index/search/service_logs:
get:
@@ -16786,7 +16819,6 @@ components:
type: string
required:
- s3
TeamsChannel:
type: object
required:
@@ -16810,4 +16842,4 @@ components:
channel_name:
type: string
description: Microsoft Teams channel name
minLength: 1
minLength: 1
backend/windmill-api/src/configs.rs
+22
View File
@@ -33,6 +33,10 @@ pub fn global_service() -> Router {
"/list_autoscaling_events/:worker_group",
get(list_autoscaling_events),
)
.route(
"/list_available_python_versions",
get(list_available_python_versions),
)
}
#[derive(Serialize, Deserialize, FromRow)]
@@ -205,6 +209,24 @@ async fn list_autoscaling_events(
Ok(Json(events))
}
async fn list_available_python_versions() -> error::JsonResult<Vec<String>> {
#[cfg(not(feature = "python"))]
return Err(error::Error::BadRequest(
"Python listing available only with 'python' feature enabled".to_string(),
));
#[cfg(feature = "python")]
use itertools::Itertools;
#[cfg(feature = "python")]
return Ok(Json(
windmill_worker::PyV::list_available_python_versions()
.await
.iter()
.map(|v| v.to_string())
.collect_vec(),
));
}
#[cfg(feature = "enterprise")]
async fn list_configs(
authed: ApiAuthed,
backend/windmill-api/src/jobs.rs
+3 -8
View File
@@ -83,8 +83,6 @@ use windmill_common::{
},
};
#[cfg(all(feature = "enterprise", feature = "parquet"))]
use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
#[cfg(feature = "prometheus")]
use windmill_common::{METRICS_DEBUG_ENABLED, METRICS_ENABLED};
@@ -1058,7 +1056,7 @@ async fn get_logs_from_store(
if log_offset > 0 {
if let Some(file_index) = log_file_index.clone() {
tracing::debug!("Getting logs from store: {file_index:?}");
if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
tracing::debug!("object store client present, streaming from there");
let logs = logs.to_string();
@@ -4962,10 +4960,7 @@ async fn run_bundle_preview_script(
uploaded = true;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
let object_store = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone();
let object_store = windmill_common::s3_helpers::get_object_store().await;
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
let object_store: Option<()> = None;
@@ -5663,7 +5658,7 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
let file = os
.get(&object_store::path::Path::from(format!("logs/{file_p}")))
.await;
backend/windmill-api/src/scripts.rs
+6 -7
View File
@@ -410,10 +410,7 @@ async fn create_snapshot_script(
uploaded = true;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
let object_store = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone();
let object_store = windmill_common::s3_helpers::get_object_store().await;
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
let object_store: Option<()> = None;
@@ -1327,10 +1324,12 @@ async fn raw_script_by_path_internal(
w_id
)
.fetch_one(&db)
.await?;
if exists.unwrap_or(false) {
.await?
.unwrap_or(false);
if exists {
return Err(Error::NotFound(format!(
"Script {path} not visible to {} but exists",
"Script {path} exists but {} does not have permissions to access it",
authed.username
)));
}
backend/windmill-api/src/service_logs.rs
+1 -4
View File
@@ -98,10 +98,7 @@ async fn get_log_file(
require_devops_role(&db, &email).await?;
let path = path.to_path();
#[cfg(feature = "parquet")]
let s3_client = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone();
let s3_client = windmill_common::s3_helpers::get_object_store().await;
#[cfg(feature = "parquet")]
if let Some(s3_client) = s3_client {
let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path);
backend/windmill-api/src/settings.rs
+4 -1
View File
@@ -120,12 +120,15 @@ use windmill_common::s3_helpers::build_object_store_from_settings;
#[cfg(feature = "parquet")]
pub async fn test_s3_bucket(
_authed: ApiAuthed,
Extension(db): Extension<DB>,
Json(test_s3_bucket): Json<ObjectSettings>,
) -> error::Result<String> {
use bytes::Bytes;
use futures::StreamExt;
let client = build_object_store_from_settings(test_s3_bucket).await?;
let client = build_object_store_from_settings(test_s3_bucket, Some(&db))
.await?
.store;
let mut list = client.list(Some(&object_store::path::Path::from("".to_string())));
let first_file = list.next().await;
backend/windmill-common/Cargo.toml
+4 -2
View File
@@ -12,14 +12,14 @@ tantivy = []
prometheus = ["dep:prometheus"]
loki = ["dep:tracing-loki"]
benchmark = []
parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts", "dep:datafusion"]
parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts", "dep:aws-smithy-types-convert", "dep:datafusion"]
aws_auth = ["dep:aws-sdk-sts", "dep:aws-config"]
otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk",
"dep:opentelemetry", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic"]
smtp = ["dep:mail-send"]
scoped_cache = []
cloud = []
openidconnect = ["dep:openidconnect"]
[lib]
name = "windmill_common"
path = "src/lib.rs"
@@ -62,6 +62,7 @@ object_store = { workspace = true, optional = true }
prometheus = { workspace = true, optional = true }
aws-config = { workspace = true, optional = true }
aws-sdk-sts = { workspace = true, optional = true }
aws-smithy-types-convert = { workspace = true, optional = true }
indexmap.workspace = true
bytes.workspace = true
mail-send = { workspace = true, optional = true }
@@ -75,6 +76,7 @@ windmill-parser-ts.workspace = true
windmill-parser-py.workspace = true
jsonwebtoken.workspace = true
backon.workspace = true
openidconnect = { workspace = true, optional = true }
strum.workspace = true
strum_macros.workspace = true
backend/windmill-common/src/client.rs
+200
View File
@@ -0,0 +1,200 @@
use anyhow::Context;
use reqwest::{Body, Response};
use serde::de::DeserializeOwned;
use crate::utils::HTTP_CLIENT;
#[derive(Clone)]
pub struct AuthedClient {
pub base_internal_url: String,
pub workspace: String,
pub token: String,
pub force_client: Option<reqwest::Client>,
}
impl AuthedClient {
pub async fn get(&self, url: &str, query: Vec<(&str, String)>) -> anyhow::Result<Response> {
self.force_client
.as_ref()
.unwrap_or(&HTTP_CLIENT)
.get(url)
.query(&query)
.header(
reqwest::header::ACCEPT,
reqwest::header::HeaderValue::from_static("application/json"),
)
.header(
reqwest::header::AUTHORIZATION,
reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))?,
)
.send()
.await
.map_err(|e| {
tracing::error!("Error executing get request from authed http client to {url} with query {query:?}: {e}");
anyhow::anyhow!("Error executing get request from authed http client to {url} with query {query:?}: {e}")
})
}
pub async fn get_id_token(&self, audience: &str) -> anyhow::Result<String> {
let url = format!(
"{}/api/w/{}/oidc/token/{}",
self.base_internal_url, self.workspace, audience
);
let response = self.get(&url, vec![]).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<String>()
.await
.context("decoding oidc token as json string")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_resource_value<T: DeserializeOwned>(&self, path: &str) -> anyhow::Result<T> {
let url = format!(
"{}/api/w/{}/resources/get_value/{}",
self.base_internal_url, self.workspace, path
);
let response = self.get(&url, vec![]).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<T>()
.await
.context("decoding resource value as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_variable_value(&self, path: &str) -> anyhow::Result<String> {
let url = format!(
"{}/api/w/{}/variables/get_value/{}",
self.base_internal_url, self.workspace, path
);
let response = self.get(&url, vec![]).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<String>()
.await
.context("decoding variable value as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_resource_value_interpolated<T: DeserializeOwned>(
&self,
path: &str,
job_id: Option<String>,
) -> anyhow::Result<T> {
let url = format!(
"{}/api/w/{}/resources/get_value_interpolated/{}",
self.base_internal_url, self.workspace, path
);
let mut query = Vec::with_capacity(1usize);
if let Some(v) = &job_id {
query.push(("job_id", v.to_string()));
}
let response = self.get(&url, query).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<T>()
.await
.context("decoding interpolated resource value as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_completed_job_result<T: DeserializeOwned>(
&self,
path: &str,
json_path: Option<String>,
) -> anyhow::Result<T> {
let url = format!(
"{}/api/w/{}/jobs_u/completed/get_result/{}",
self.base_internal_url, self.workspace, path
);
let query = if let Some(json_path) = json_path {
vec![("json_path", json_path)]
} else {
vec![]
};
let response = self.get(&url, query).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<T>()
.await
.context("decoding completed job result as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_result_by_id<T: DeserializeOwned>(
&self,
flow_job_id: &str,
node_id: &str,
json_path: Option<String>,
) -> anyhow::Result<T> {
let url = format!(
"{}/api/w/{}/jobs/result_by_id/{}/{}",
self.base_internal_url, self.workspace, flow_job_id, node_id
);
let query = if let Some(json_path) = json_path {
vec![("json_path", json_path)]
} else {
vec![]
};
let response = self.get(&url, query).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<T>()
.await
.context("decoding result by id as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn upload_s3_file<S>(
&self,
workspace_id: &str,
object_key: String,
storage: Option<String>,
body: S,
) -> anyhow::Result<()>
where
S: futures::stream::TryStream + Send + 'static,
S::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
bytes::Bytes: From<S::Ok>,
{
let mut query = vec![("file_key", object_key)];
if let Some(storage) = storage {
query.push(("storage", storage));
}
let response = self
.force_client
.as_ref()
.unwrap_or(&HTTP_CLIENT)
.post(format!(
"{}/api/w/{}/job_helpers/upload_s3_file",
self.base_internal_url, workspace_id
))
.query(&query)
.header(
reqwest::header::ACCEPT,
reqwest::header::HeaderValue::from_static("application/json"),
)
.header(
reqwest::header::AUTHORIZATION,
reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))
.map_err(|e| anyhow::anyhow!(e.to_string()))?,
)
.body(Body::wrap_stream(body))
.send()
.await
.context(format!("Sent upload_s3_file request",))
.map_err(|e| anyhow::anyhow!(e.to_string()))?;
match response.status().as_u16() {
200u16 => Ok(()),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?,
}
}
}
backend/windmill-common/src/global_settings.rs
+1 -1
View File
@@ -30,7 +30,7 @@ pub const EXPOSE_METRICS_SETTING: &str = "expose_metrics";
pub const EXPOSE_DEBUG_METRICS_SETTING: &str = "expose_debug_metrics";
pub const KEEP_JOB_DIR_SETTING: &str = "keep_job_dir";
pub const REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING: &str = "require_preexisting_user_for_oauth";
pub const OBJECT_STORE_CACHE_CONFIG_SETTING: &str = "object_store_cache_config";
pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config";
pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation";
pub const HUB_BASE_URL_SETTING: &str = "hub_base_url";
backend/windmill-common/src/job_s3_helpers_ee.rs
+29 -13
View File
@@ -1,18 +1,34 @@
use std::future::Future;
use crate::s3_helpers::{ObjectStoreResource, StorageResourceType};
use crate::{
error::Error,
s3_helpers::{ObjectStoreResource, StorageResourceType},
};
pub async fn get_s3_resource_internal<'c, F, Fut>(
pub async fn get_s3_resource_internal<'c>(
_resource_type: StorageResourceType,
_s3_resource_value_raw: serde_json::Value,
_gen_token: F,
) -> crate::error::Result<ObjectStoreResource>
where
F: FnOnce(String) -> Fut,
Fut: Future<Output = Result<String, Error>> + Send + 'static,
{
_gen_token: TokenGenerator<'c>,
_db: &crate::DB,
) -> crate::error::Result<ObjectStoreResource> {
todo!()
}
pub enum TokenGenerator<'c> {
AsClient(&'c crate::client::AuthedClient),
AsServerInstance(),
}
impl<'c> TokenGenerator<'c> {
pub async fn gen_token(
&self,
_audience: &str,
_db: Option<&crate::DB>,
) -> anyhow::Result<String> {
todo!()
}
}
#[cfg(feature = "parquet")]
pub(crate) async fn generate_s3_aws_oidc_resource<'c>(
_clone: crate::s3_helpers::S3AwsOidcResource,
_token_generator: TokenGenerator<'c>,
_init_private_key: Option<&sqlx::Pool<sqlx::Postgres>>,
) -> crate::error::Result<ObjectStoreResource> {
todo!()
}
backend/windmill-common/src/jobs.rs
+2 -2
View File
@@ -608,11 +608,11 @@ pub async fn get_logs_from_store(
logs: &str,
log_file_index: &Option<Vec<String>>,
) -> Option<impl Stream<Item = Result<Bytes, object_store::Error>>> {
use crate::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
use crate::s3_helpers::get_object_store;
if log_offset > 0 {
if let Some(file_index) = log_file_index.clone() {
if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
if let Some(os) = get_object_store().await {
let logs = logs.to_string();
let stream = async_stream::stream! {
for file_p in file_index.clone() {
backend/windmill-common/src/lib.rs
+4
View File
@@ -30,6 +30,7 @@ pub mod auth;
#[cfg(feature = "benchmark")]
pub mod bench;
pub mod cache;
pub mod client;
pub mod db;
pub mod ee;
pub mod email_ee;
@@ -43,6 +44,9 @@ pub mod job_metrics;
#[cfg(feature = "parquet")]
pub mod job_s3_helpers_ee;
#[cfg(all(feature = "enterprise", feature = "openidconnect"))]
pub mod oidc_ee;
pub mod jobs;
pub mod jwt;
pub mod more_serde;
backend/windmill-common/src/oidc_ee.rs
+198
View File
@@ -0,0 +1,198 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2023
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use serde::{Deserialize, Serialize};
use tokio::sync::RwLock;
#[cfg(all(feature = "enterprise", feature = "openidconnect"))]
use {
crate::db::DB,
crate::{auth::IdToken as WindmillIdToken, error::Result},
anyhow,
openidconnect::{
core::{CoreJwsSigningAlgorithm, CoreRsaPrivateSigningKey},
IssuerUrl, JsonWebKeyId,
},
std::process::Command,
};
#[cfg(feature = "openidconnect")]
use openidconnect::AdditionalClaims;
#[cfg(feature = "openidconnect")]
impl AdditionalClaims for JobClaim {}
#[cfg(feature = "openidconnect")]
impl AdditionalClaims for WorkspaceClaim {}
#[cfg(feature = "openidconnect")]
impl AdditionalClaims for InstanceClaim {}
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
pub struct WorkspaceClaim {
pub workspace: String,
}
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
pub struct InstanceClaim {}
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
pub struct JobClaim {
pub job_id: String,
pub path: Option<String>,
pub flow_path: Option<String>,
pub groups: Vec<String>,
pub username: String,
pub email: String,
pub workspace: String,
}
lazy_static::lazy_static! {
static ref PRIVATE_KEY: RwLock<Option<String>> = RwLock::new(None);
}
pub async fn generate_id_token<T: AdditionalClaims>(
db: Option<&DB>,
claim: T,
audience: &str,
identifier: String,
email: Option<String>,
) -> Result<WindmillIdToken> {
use chrono::{Duration, Utc};
use openidconnect::{
core::{CoreGenderClaim, CoreJweContentEncryptionAlgorithm},
Audience, EndUserEmail, IdToken, IdTokenClaims, StandardClaims, SubjectIdentifier,
};
let private_key = get_private_key(db).await?;
let issue_url = format!("{}/api/oidc/", crate::BASE_URL.read().await.clone());
let issue_time = Utc::now();
let expiration = issue_time + Duration::try_hours(48).unwrap();
let id_token = IdToken::<
T,
CoreGenderClaim,
CoreJweContentEncryptionAlgorithm,
CoreJwsSigningAlgorithm,
>::new(
IdTokenClaims::<T, CoreGenderClaim>::new(
// Specify the issuer URL for the OpenID Connect Provider.
IssuerUrl::new(issue_url)
.map_err(|e| anyhow::anyhow!("Failed to generate IssueUrl: {}", e))?,
// The audience is usually a single entry with the client ID of the client for whom
// the ID token is intended. This is a required claim.
vec![Audience::new(audience.to_string())],
// The ID token expiration is usually much shorter than that of the access or refresh
// tokens issued to clients.
expiration,
// The issue time is usually the current time.
issue_time,
// Set the standard claims defined by the OpenID Connect Core spec.
StandardClaims::new(
// Stable subject identifiers are recommended in place of e-mail addresses or other
// potentially unstable identifiers. This is the only required claim.
SubjectIdentifier::new(identifier),
)
// Optional: specify the user's e-mail address. This should only be provided if the
// client has been granted the 'profile' or 'email' scopes.
.set_email(email.map(|x| EndUserEmail::new(x)))
// Optional: specify whether the provider has verified the user's e-mail address.
.set_email_verified(Some(true)),
// OpenID Connect Providers may supply custom claims by providing a struct that
// implements the AdditionalClaims trait. This requires manually using the
// generic IdTokenClaims struct rather than the CoreIdTokenClaims type alias,
// however.
claim,
),
// The private key used for signing the ID token. For confidential clients (those able
// to maintain a client secret), a CoreHmacKey can also be used, in conjunction
// with one of the CoreJwsSigningAlgorithm::HmacSha* signing algorithms. When using an
// HMAC-based signing algorithm, the UTF-8 representation of the client secret should
// be used as the HMAC key.
&CoreRsaPrivateSigningKey::from_pem(
&private_key,
Some(JsonWebKeyId::new("windmill".to_string())),
)
.map_err(|e| anyhow::anyhow!("Invalid private key: {}", e))?,
// Uses the RS256 signature algorithm. This crate supports any RS*, PS*, or HS*
// signature algorithm.
CoreJwsSigningAlgorithm::RsaSsaPkcs1V15Sha256,
// When returning the ID token alongside an access token (e.g., in the Authorization Code
// flow), it is recommended to pass the access token here to set the `at_hash` claim
// automatically.
None,
// When returning the ID token alongside an authorization code (e.g., in the implicit
// flow), it is recommended to pass the authorization code here to set the `c_hash` claim
// automatically.
None,
)
.map_err(|e| anyhow::anyhow!("Failed to generate token: {}", e))?;
Ok(WindmillIdToken::new(id_token.to_string(), expiration))
}
#[cfg(all(feature = "enterprise", feature = "openidconnect"))]
pub async fn get_private_key(db: Option<&DB>) -> anyhow::Result<String> {
if let Some(key) = PRIVATE_KEY.read().await.clone() {
return Ok(key);
} else if let Some(db) = db {
let key = sqlx::query_scalar!(
"SELECT value->>'private_key' FROM global_settings WHERE name = 'rsa_keys'",
)
.fetch_optional(db)
.await?
.flatten();
let key = key.filter(|s| !s.is_empty());
if let Some(key) = key {
return Ok(key);
} else {
let keys = gen_pems(db).await?;
return Ok(keys.private_key);
}
} else {
return Err(anyhow::anyhow!("Private key not found and no db provided"));
}
}
#[cfg(all(feature = "enterprise", feature = "openidconnect"))]
#[derive(Debug, Clone, serde::Serialize)]
struct Keys {
private_key: String,
}
#[cfg(all(feature = "enterprise", feature = "openidconnect"))]
async fn gen_pems(db: &DB) -> anyhow::Result<Keys> {
use anyhow::anyhow;
let private_key_cmd = Command::new("openssl")
.arg("genrsa")
.arg("--traditional")
.arg("2048")
.output()
.expect("failed to execute process");
let private_key = String::from_utf8(private_key_cmd.stdout)?;
tracing::debug!("Generated private key: {}", private_key);
if private_key.is_empty() {
return Err(anyhow!("Failed to generate RSA key: key is empty"));
}
let keys = Keys { private_key };
sqlx::query!(
r#"INSERT INTO global_settings (name, value) VALUES ('rsa_keys', $1)"#,
serde_json::to_value(&keys).unwrap()
)
.execute(db)
.await?;
Ok(keys)
}
backend/windmill-common/src/s3_helpers.rs
+202 -6
View File
@@ -4,6 +4,7 @@ use crate::error;
use aws_sdk_sts::config::ProvideCredentials;
#[cfg(feature = "parquet")]
use axum::async_trait;
use chrono::{DateTime, Utc};
#[cfg(feature = "parquet")]
use object_store::aws::AwsCredential;
#[cfg(feature = "parquet")]
@@ -17,6 +18,7 @@ use reqwest::header::HeaderMap;
use serde::{Deserialize, Serialize};
#[cfg(feature = "parquet")]
use std::sync::{Arc, Mutex};
#[cfg(feature = "parquet")]
use tokio::sync::RwLock;
@@ -46,9 +48,170 @@ use tokio::task;
use windmill_parser_sql::S3ModeFormat;
#[cfg(feature = "parquet")]
lazy_static::lazy_static! {
#[derive(Clone)]
pub struct ExpirableObjectStore {
pub store: Arc<dyn ObjectStore>,
pub refresh: Option<ObjectStoreRefresh>,
}
pub static ref OBJECT_STORE_CACHE_SETTINGS: Arc<RwLock<Option<Arc<dyn ObjectStore>>>> = Arc::new(RwLock::new(None));
#[cfg(feature = "parquet")]
#[derive(Clone)]
pub struct ObjectStoreRefresh {
refresh: Option<DateTime<Utc>>,
settings: ObjectSettings,
}
#[cfg(feature = "parquet")]
impl ObjectStoreRefresh {
pub fn new(settings: ObjectSettings, refresh: Option<DateTime<Utc>>) -> Self {
Self { settings, refresh }
}
fn refresh_needed(&self) -> bool {
if let Some(refresh) = self.refresh {
if refresh < Utc::now() - chrono::Duration::minutes(1) {
return true;
}
}
return false;
}
async fn refresh(&self) -> Option<ExpirableObjectStore> {
return build_object_store_from_settings(self.settings.clone(), None)
.await
.map_err(|e| {
tracing::error!("Error building s3 client from settings: {:?}", e);
e
})
.ok();
}
}
#[cfg(feature = "parquet")]
impl From<Arc<dyn ObjectStore>> for ExpirableObjectStore {
fn from(store: Arc<dyn ObjectStore>) -> Self {
Self { store, refresh: None }
}
}
// #[cfg(feature = "parquet")]
// impl ExpirableObjectStore {
// pub fn new(store: Arc<dyn ObjectStore>, expiration: Option<DateTime<Utc>>) -> Self {
// Self { store, expiration }
// }
// }
#[cfg(feature = "parquet")]
lazy_static::lazy_static! {
pub static ref OBJECT_STORE_SETTINGS: Arc<RwLock<Option<ExpirableObjectStore>>> = Arc::new(RwLock::new(None));
}
#[cfg(feature = "parquet")]
pub async fn get_object_store() -> Option<Arc<dyn ObjectStore>> {
let settings = OBJECT_STORE_SETTINGS.read().await;
if let Some(s) = settings.as_ref() {
match &s.refresh {
Some(refresh) => {
if refresh.refresh_needed() {
let refresh = refresh.clone();
drop(settings);
let new_store = refresh.refresh().await;
if let Some(new_store) = new_store {
let mut s3_cache_settings = OBJECT_STORE_SETTINGS.write().await;
let arc = new_store.store.clone();
*s3_cache_settings = Some(new_store);
return Some(arc);
} else {
return None;
}
} else {
return Some(s.store.clone());
}
}
None => {
return Some(s.store.clone());
}
}
} else {
return None;
}
}
#[cfg(feature = "parquet")]
pub enum ObjectStoreReload {
//if the jwks endpoints are not up yet, we should retry later soon
Later,
Never,
}
#[cfg(feature = "parquet")]
pub async fn reload_object_store_setting(db: &crate::DB) -> ObjectStoreReload {
use crate::{
ee::{get_license_plan, LicensePlan},
global_settings::{load_value_from_global_settings, OBJECT_STORE_CONFIG_SETTING},
s3_helpers::ObjectSettings,
};
let s3_config = load_value_from_global_settings(db, OBJECT_STORE_CONFIG_SETTING).await;
if let Err(e) = s3_config {
tracing::error!("Error reloading s3 cache config: {:?}", e)
} else {
if let Some(v) = s3_config.unwrap() {
if matches!(get_license_plan().await, LicensePlan::Pro) {
tracing::error!("S3 cache is not available for pro plan");
return ObjectStoreReload::Never;
}
let setting = serde_json::from_value::<ObjectSettings>(v);
match setting {
Ok(setting) => {
let is_oidc = matches!(setting, ObjectSettings::AwsOidc(_));
let s3_client = build_object_store_from_settings(setting, Some(db)).await;
match s3_client {
Ok(s3_client) => {
let mut s3_cache_settings = OBJECT_STORE_SETTINGS.write().await;
*s3_cache_settings = Some(s3_client);
}
Err(e) => {
if is_oidc {
tracing::error!("Error building s3 client from oidc settings. It may be due to the jwks endpoints not being up yet, it will be attempted again in 10s to leave time for the server to be ready: {:?}", e);
return ObjectStoreReload::Later;
} else {
tracing::error!("Error building s3 client from settings: {:?}", e);
}
}
}
}
Err(e) => {
tracing::error!("Error parsing s3 cache config: {:?}", e)
}
}
} else {
let mut s3_cache_settings = OBJECT_STORE_SETTINGS.write().await;
if std::env::var("S3_CACHE_BUCKET").is_ok() {
if matches!(get_license_plan().await, LicensePlan::Pro) {
tracing::error!("S3 cache is not available for pro plan");
return ObjectStoreReload::Never;
}
*s3_cache_settings = build_s3_client_from_settings(S3Settings {
bucket: None,
region: None,
access_key: None,
secret_key: None,
endpoint: None,
store_logs: None,
path_style: None,
allow_http: None,
port: None,
})
.await
.ok()
.map(|x| ExpirableObjectStore::from(x))
} else {
*s3_cache_settings = None;
}
}
}
return ObjectStoreReload::Never;
}
#[derive(Serialize, Deserialize, Debug)]
@@ -81,6 +244,15 @@ pub enum ObjectStoreResource {
Azure(AzureBlobResource),
}
impl ObjectStoreResource {
pub fn expiration(&self) -> Option<DateTime<Utc>> {
match self {
ObjectStoreResource::S3(s3_resource) => s3_resource.expiration,
_ => None,
}
}
}
#[derive(Deserialize, Debug)]
pub enum StorageResourceType {
S3,
@@ -104,6 +276,8 @@ pub struct S3Resource {
#[serde(rename = "pathStyle")]
pub path_style: Option<bool>,
pub token: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub expiration: Option<DateTime<Utc>>,
pub port: Option<u16>,
}
@@ -126,7 +300,7 @@ pub struct AzureBlobResource {
pub federated_token_file: Option<String>,
}
#[derive(Debug, Deserialize, Serialize, Clone)]
#[derive(Debug, Deserialize, Serialize, Clone, Hash)]
pub struct S3AwsOidcResource {
#[serde(rename = "bucket")]
pub bucket: String,
@@ -412,6 +586,7 @@ pub enum ObjectStoreSettings {
pub enum ObjectSettings {
S3(S3Settings),
Azure(AzureBlobResource),
AwsOidc(S3AwsOidcResource),
}
impl ObjectSettings {
@@ -419,6 +594,7 @@ impl ObjectSettings {
match self {
ObjectSettings::S3(s3_settings) => s3_settings.bucket.as_ref(),
ObjectSettings::Azure(azure_settings) => Some(&azure_settings.container_name),
ObjectSettings::AwsOidc(s3_aws_oidc_settings) => Some(&s3_aws_oidc_settings.bucket),
}
}
}
@@ -426,12 +602,31 @@ impl ObjectSettings {
#[cfg(feature = "parquet")]
pub async fn build_object_store_from_settings(
settings: ObjectSettings,
) -> error::Result<Arc<dyn ObjectStore>> {
init_private_key: Option<&crate::DB>,
) -> error::Result<ExpirableObjectStore> {
match settings {
ObjectSettings::S3(s3_settings) => build_s3_client_from_settings(s3_settings).await,
ObjectSettings::S3(s3_settings) => build_s3_client_from_settings(s3_settings)
.await
.map(|x| ExpirableObjectStore::from(x)),
ObjectSettings::Azure(azure_settings) => {
let azure_blob_resource = azure_settings;
build_azure_blob_client(&azure_blob_resource)
build_azure_blob_client(&azure_blob_resource).map(|x| ExpirableObjectStore::from(x))
}
ObjectSettings::AwsOidc(ref s3_aws_oidc_settings) => {
let token_generator = crate::job_s3_helpers_ee::TokenGenerator::AsServerInstance();
let res = crate::job_s3_helpers_ee::generate_s3_aws_oidc_resource(
s3_aws_oidc_settings.clone(),
token_generator,
init_private_key,
)
.await?;
build_object_store_client(&res)
.await
.map(|x| ExpirableObjectStore {
store: x,
refresh: Some(ObjectStoreRefresh::new(settings.clone(), res.expiration())),
})
}
}
}
@@ -479,6 +674,7 @@ pub async fn build_s3_client_from_settings(
path_style: settings.path_style,
port: settings.port,
token: None,
expiration: None,
};
build_s3_client(&s3_resource).await
backend/windmill-common/src/worker.rs
+4 -15
View File
@@ -516,6 +516,7 @@ fn parse_file<T: FromStr>(path: &str) -> Option<T> {
pub struct PythonAnnotations {
pub no_cache: bool,
pub no_postinstall: bool,
pub py_select_latest: bool,
pub skip_result_postprocessing: bool,
pub py310: bool,
pub py311: bool,
@@ -581,11 +582,7 @@ pub async fn load_cache(bin_path: &str, _remote_path: &str, is_dir: bool) -> (bo
(true, format!("loaded from local cache: {}\n", bin_path))
} else {
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if let Some(os) = crate::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone()
{
if let Some(os) = crate::s3_helpers::get_object_store().await {
let started = std::time::Instant::now();
use crate::s3_helpers::attempt_fetch_bytes;
@@ -628,11 +625,7 @@ pub async fn exists_in_cache(bin_path: &str, _remote_path: &str) -> bool {
return true;
} else {
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if let Some(os) = crate::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone()
{
if let Some(os) = crate::s3_helpers::get_object_store().await {
return os
.get(&object_store::path::Path::from(_remote_path))
.await
@@ -650,11 +643,7 @@ pub async fn save_cache(
) -> crate::error::Result<String> {
let mut _cached_to_s3 = false;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if let Some(os) = crate::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone()
{
if let Some(os) = crate::s3_helpers::get_object_store().await {
use object_store::path::Path;
let file_to_cache = if is_dir {
let tar_path = format!(
backend/windmill-queue/src/jobs.rs
+11 -1
View File
@@ -980,6 +980,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
is_flow_step = queued_job.is_flow_step(),
language = ?queued_job.script_lang,
scheduled_for = ?queued_job.scheduled_for,
workspace_id = ?queued_job.workspace_id,
success,
"inserted completed job: {} (success: {success})",
queued_job.id
@@ -2122,10 +2123,19 @@ pub struct PulledJob {
pub permissioned_as_folders: Option<Vec<serde_json::Value>>,
}
// NOTE:
// Precomputed by the server
// Used to offload work from agent workers to server
#[derive(Serialize, Deserialize)]
pub enum PrecomputedAgentInfo {
Bun { local: String, remote: String },
Python { py_version: Option<u32>, requirements: Option<String> },
Python {
// V1, not used anymore. Exists for compat.
// TODO: Needs to be removed eventually
py_version: Option<u32>,
py_version_v2: Option<String>,
requirements: Option<String> },
}
#[derive(Serialize, Deserialize)]
backend/windmill-worker/Cargo.toml
+1
View File
@@ -116,6 +116,7 @@ convert_case.workspace = true
yaml-rust.workspace = true
backon.workspace = true
winapi = { workspace = true, optional = true }
pep440_rs.workspace = true
opentelemetry = { workspace = true, optional = true }
bollard = { workspace = true, optional = true }
backend/windmill-worker/src/ansible_executor.rs
+9 -11
View File
@@ -30,10 +30,11 @@ use crate::{
start_child_process, transform_json, OccupancyMetrics,
},
handle_child::handle_child,
python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion},
AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV,
PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV,
python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile},
PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
PY_INSTALL_DIR, TZ_ENV,
};
use windmill_common::client::AuthedClient;
lazy_static::lazy_static! {
static ref ANSIBLE_PLAYBOOK_PATH: String =
@@ -373,7 +374,7 @@ async fn handle_ansible_python_deps(
worker_name,
w_id,
&mut Some(occupancy_metrics),
PyVersion::Py311,
PyVAlias::Py311.into(),
false,
)
.await
@@ -387,10 +388,7 @@ async fn handle_ansible_python_deps(
if requirements.len() > 0 {
let mut venv_path = handle_python_reqs(
requirements
.split("\n")
.filter(|x| !x.starts_with("--"))
.collect(),
crate::python_executor::split_requirements(requirements),
job_id,
w_id,
mem_peak,
@@ -400,7 +398,7 @@ async fn handle_ansible_python_deps(
job_dir,
worker_dir,
&mut Some(occupancy_metrics),
crate::python_executor::PyVersion::Py311,
PyVAlias::default().into(),
)
.await?;
additional_python_paths.append(&mut venv_path);
@@ -1193,7 +1191,7 @@ async fn create_file_resources(
job_dir: &str,
args: Option<&HashMap<String, Box<RawValue>>>,
r: &AnsibleRequirements,
client: &crate::AuthedClient,
client: &AuthedClient,
conn: &Connection,
) -> error::Result<Vec<String>> {
let mut logs = String::new();
@@ -1270,7 +1268,7 @@ async fn create_file_resources(
}
async fn get_resource_or_variable_content(
client: &crate::AuthedClient,
client: &AuthedClient,
path: &ResourceOrVariablePath,
job_id: String,
) -> anyhow::Result<String> {
backend/windmill-worker/src/bash_executor.rs
+37 -13
View File
@@ -43,9 +43,11 @@ use crate::{
OccupancyMetrics,
},
handle_child::handle_child,
AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
POWERSHELL_CACHE_DIR, POWERSHELL_PATH, PROXY_ENVS, TZ_ENV,
};
use windmill_common::client::AuthedClient;
#[cfg(windows)]
use crate::SYSTEM_ROOT;
@@ -299,16 +301,28 @@ async fn handle_docker_job(
}
let wait_f = async {
let wait = client
let waited = client
.wait_container::<String>(&container_id, None)
.try_collect::<Vec<_>>()
.await
.map_err(|e| {
.await;
match waited {
Ok(wait) => Ok(wait.first().map(|x| x.status_code)),
Err(bollard::errors::Error::DockerResponseServerError { status_code, message }) => {
append_logs(&job_id, &workspace_id, &format!(": {message}"), conn).await;
Ok(Some(status_code as i64))
}
Err(bollard::errors::Error::DockerContainerWaitError { error, code }) => {
append_logs(&job_id, &workspace_id, &format!("{error}"), conn).await;
Ok(Some(code as i64))
}
Err(e) => {
tracing::error!("Error waiting for container: {:?}", e);
anyhow::anyhow!("Error waiting for container: {:?}", e)
})?;
let waited = wait.first().map(|x| x.status_code);
Ok(waited)
Err(Error::ExecutionErr(format!(
"Error waiting for container: {:?}",
e
)))
}
}
};
let ncontainer_id = container_id.to_string();
@@ -317,7 +331,7 @@ async fn handle_docker_job(
let conn2 = conn.clone();
let worker_name2 = worker_name.to_string();
let (tx, mut rx) = tokio::sync::broadcast::channel::<()>(1);
let workspace_id2 = workspace_id.to_string();
let mut killpill_rx = killpill_rx.resubscribe();
let logs = tokio::spawn(async move {
let client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow);
@@ -332,6 +346,13 @@ async fn handle_docker_job(
..Default::default()
}),
);
append_logs(
&job_id,
&workspace_id2,
"\ndocker logs stream started\n",
&conn2,
)
.await;
loop {
tokio::select! {
log = log_stream.next() => {
@@ -441,11 +462,14 @@ async fn handle_docker_job(
let result = result.unwrap();
if result.is_some_and(|x| x > 0) {
return Err(Error::ExecutionErr(format!(
"Docker job completed with unsuccessful exit status: {}",
result.unwrap()
)));
}
return Ok(to_raw_value(&json!(format!(
"Docker exit status: {}",
result
.map(|x| x.to_string())
.unwrap_or_else(|| "none".to_string())
"Docker job completed with success exit status"
))));
}
backend/windmill-worker/src/bigquery_executor.rs
+2 -4
View File
@@ -4,6 +4,7 @@ use futures::future::BoxFuture;
use futures::{FutureExt, StreamExt};
use reqwest::Client;
use serde_json::{json, value::RawValue, Value};
use windmill_common::client::AuthedClient;
use windmill_common::error::to_anyhow;
use windmill_common::s3_helpers::convert_json_line_stream;
use windmill_common::worker::Connection;
@@ -16,15 +17,12 @@ use windmill_queue::CanceledBy;
use serde::Deserialize;
use crate::common::{build_args_values, resolve_job_timeout};
use crate::common::{
build_http_client, s3_mode_args_to_worker_data, OccupancyMetrics, S3ModeWorkerData,
};
use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
use crate::{
common::{build_args_values, resolve_job_timeout},
AuthedClient,
};
use gcp_auth::{AuthenticationManager, CustomServiceAccount};
backend/windmill-worker/src/bun_executor.rs
+5 -7
View File
@@ -20,10 +20,11 @@ use crate::{
read_file_content, read_result, start_child_process, write_file_binary, OccupancyMetrics,
},
handle_child::handle_child,
AuthedClient, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_PATH,
DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, NPM_CONFIG_REGISTRY,
NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TZ_ENV,
BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL,
DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH,
PATH_ENV, PROXY_ENVS, TZ_ENV,
};
use windmill_common::client::AuthedClient;
#[cfg(windows)]
use crate::SYSTEM_ROOT;
@@ -612,10 +613,7 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> {
extract_saved_codebase(job_dir, &bun_cache_path, is_tar, &dst, false)?;
} else {
#[cfg(all(feature = "enterprise", feature = "parquet"))]
let object_store = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone();
let object_store = windmill_common::s3_helpers::get_object_store().await;
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
let object_store: Option<()> = None;
backend/windmill-worker/src/common.rs
+15 -31
View File
@@ -44,10 +44,8 @@ use windmill_common::{variables, DB};
use tokio::{io::AsyncWriteExt, process::Child, time::Instant};
use crate::agent_workers::UPDATE_PING_URL;
use crate::{
AuthedClient, DISABLE_NSJAIL, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION,
PATH_ENV,
};
use crate::{DISABLE_NSJAIL, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, PATH_ENV};
use windmill_common::client::AuthedClient;
pub async fn build_args_map<'a>(
job: &'a MiniPulledJob,
@@ -782,19 +780,17 @@ async fn get_workspace_s3_resource_path(
}
};
let client2 = client.clone();
let token_fn = |audience: String| async move {
client2
.get_id_token(&audience)
.await
.map_err(|e| windmill_common::error::Error::from(e))
};
let s3_resource_value_raw = client
.get_resource_value::<serde_json::Value>(path.as_str())
.await?;
get_s3_resource_internal(rt, s3_resource_value_raw, token_fn)
.await
.map(Some)
get_s3_resource_internal(
rt,
s3_resource_value_raw,
windmill_common::job_s3_helpers_ee::TokenGenerator::AsClient(client),
db,
)
.await
.map(Some)
}
#[cfg(feature = "parquet")]
@@ -1109,7 +1105,7 @@ pub async fn par_install_language_dependencies<'a>(
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
if windmill_common::s3_helpers::OBJECT_STORE_SETTINGS
.read()
.await
.is_none()
@@ -1264,11 +1260,7 @@ pub async fn par_install_language_dependencies<'a>(
#[cfg(all(feature = "enterprise", feature = "parquet"))]
let s3_pull_future = if is_not_pro {
if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone()
{
if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
Some(crate::global_cache::pull_from_tar(
os,
path.clone(),
@@ -1449,11 +1441,7 @@ pub async fn par_install_language_dependencies<'a>(
};
#[cfg(all(feature = "enterprise", feature = "parquet"))]
{
if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone()
{
if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
tokio::spawn(async move {
if let Err(e) = crate::global_cache::build_tar_and_push(
os,
@@ -1541,11 +1529,7 @@ pub async fn par_install_language_dependencies<'a>(
};
#[cfg(all(feature = "enterprise", feature = "parquet"))]
{
if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone()
{
if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
let language_name = language_name.to_owned();
tokio::spawn(async move {
if let Err(e) = crate::global_cache::build_tar_and_push(
@@ -1591,7 +1575,7 @@ pub struct S3ModeWorkerData {
}
impl S3ModeWorkerData {
pub async fn upload<S>(&self, stream: S) -> error::Result<()>
pub async fn upload<S>(&self, stream: S) -> anyhow::Result<()>
where
S: futures::stream::TryStream + Send + 'static,
S::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
backend/windmill-worker/src/csharp_executor.rs
+1 -1
View File
@@ -36,7 +36,7 @@ use crate::{
};
use crate::common::OccupancyMetrics;
use crate::AuthedClient;
use windmill_common::client::AuthedClient;
#[cfg(windows)]
use crate::SYSTEM_ROOT;
backend/windmill-worker/src/deno_executor.rs
+3 -1
View File
@@ -11,9 +11,11 @@ use crate::{
start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
AuthedClient, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY,
DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY,
PATH_ENV, TZ_ENV,
};
use windmill_common::client::AuthedClient;
use tokio::{fs::File, io::AsyncReadExt, process::Command};
use windmill_common::{error::Result, worker::write_file, BASE_URL};
use windmill_common::{
backend/windmill-worker/src/global_cache.rs
+4 -1
View File
@@ -22,6 +22,7 @@ pub async fn build_tar_and_push(
platform_agnostic: bool,
) -> error::Result<()> {
use object_store::path::Path;
use tokio::fs::create_dir_all;
use crate::TAR_PYBASE_CACHE_DIR;
@@ -36,7 +37,9 @@ pub async fn build_tar_and_push(
};
let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", lang);
let tar_path = format!("{prefix}/{folder_name}_tar.tar",);
let tar_path = format!("{prefix}/{folder_name}_tar.tar");
create_dir_all(prefix).await?;
let tar_file = std::fs::File::create(&tar_path)?;
let mut tar = tar::Builder::new(tar_file);
backend/windmill-worker/src/go_executor.rs
+4 -3
View File
@@ -19,9 +19,10 @@ use crate::{
start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR,
GO_CACHE_DIR, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV,
DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR, GO_CACHE_DIR, HOME_ENV,
NSJAIL_PATH, PATH_ENV, TZ_ENV,
};
use windmill_common::client::AuthedClient;
const GO_REQ_SPLITTER: &str = "//go.sum\n";
const NSJAIL_CONFIG_RUN_GO_CONTENT: &str = include_str!("../nsjail/run.go.config.proto");
@@ -473,7 +474,7 @@ pub async fn install_go_dependencies(
if non_dep_job {
if let Some(db) = conn.as_sql() {
sqlx::query!(
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('5 mins')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
hash,
req_content
)
backend/windmill-worker/src/graphql_executor.rs
+2 -1
View File
@@ -12,7 +12,8 @@ use serde::Deserialize;
use crate::common::{build_http_client, resolve_job_timeout, OccupancyMetrics};
use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::{common::build_args_map, AuthedClient};
use crate::common::build_args_map;
use windmill_common::client::AuthedClient;
#[derive(Deserialize)]
struct GraphqlApi {
backend/windmill-worker/src/handle_child.rs
+6 -4
View File
@@ -134,7 +134,7 @@ pub async fn handle_child(
let (tx, rx) = broadcast::channel::<()>(3);
let mut rx2: broadcast::Receiver<()> = tx.subscribe();
let output = child_joined_output_stream(&mut child, job_id.clone());
let output = child_joined_output_stream(&mut child, job_id.clone(), w_id.to_string());
let job_id: Uuid = job_id.clone();
@@ -729,6 +729,7 @@ where
fn child_joined_output_stream(
child: &mut Child,
job_id: Uuid,
w_id: String,
) -> impl stream::FusedStream<Item = io::Result<String>> {
let stderr = child
.stderr
@@ -743,8 +744,8 @@ fn child_joined_output_stream(
let stdout = BufReader::new(stdout).lines();
let stderr = BufReader::new(stderr).lines();
stream::select(
lines_to_stream(stderr, true, job_id.clone()),
lines_to_stream(stdout, false, job_id),
lines_to_stream(stderr, true, job_id.clone(), w_id.clone()),
lines_to_stream(stdout, false, job_id, w_id),
)
}
@@ -752,11 +753,12 @@ pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
mut lines: tokio::io::Lines<R>,
stderr: bool,
job_id: Uuid,
w_id: String,
) -> impl futures::Stream<Item = io::Result<String>> {
stream::poll_fn(move |cx| {
std::pin::Pin::new(&mut lines)
.poll_next_line(cx)
.map(|result| process_streaming_log_lines(result, stderr, &job_id))
.map(|result| process_streaming_log_lines(result, stderr, &job_id, &w_id))
})
}
backend/windmill-worker/src/java_executor.rs
+4 -2
View File
@@ -24,9 +24,11 @@ use crate::{
create_args_and_out_file, get_reserved_variables, par_install_language_dependencies,
read_result, start_child_process, OccupancyMetrics, RequiredDependency,
},
handle_child, AuthedClient, COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR,
handle_child, COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR,
JAVA_REPOSITORY_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
};
use windmill_common::client::AuthedClient;
lazy_static::lazy_static! {
static ref JAVA_CONCURRENT_DOWNLOADS: usize = std::env::var("JAVA_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(20)).unwrap_or(20);
static ref JAVA_PATH: String = std::env::var("JAVA_PATH").unwrap_or_else(|_| "/usr/bin/java".to_string());
@@ -243,7 +245,7 @@ pub async fn resolve<'a>(
if let Connection::Sql(db) = conn {
sqlx::query!(
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('5 mins')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
req_hash,
lock.clone(),
)
backend/windmill-worker/src/job_logger_ee.rs
+1
View File
@@ -36,6 +36,7 @@ pub(crate) fn process_streaming_log_lines(
r: Result<Option<String>, io::Error>,
_stderr: bool,
_job_id: &Uuid,
_w_id: &str,
) -> Option<Result<String, io::Error>> {
r.transpose()
}
backend/windmill-worker/src/js_eval.rs
+2 -1
View File
@@ -48,7 +48,8 @@ use windmill_common::worker::{write_file, TMP_DIR};
use windmill_common::flow_status::JobResult;
use windmill_queue::CanceledBy;
use crate::{common::OccupancyMetrics, AuthedClient};
use crate::common::OccupancyMetrics;
use windmill_common::client::AuthedClient;
#[cfg(feature = "deno_core")]
use crate::{common::unsafe_raw, handle_child::run_future_with_polling_update_job_poller};
backend/windmill-worker/src/lib.rs
+5
View File
@@ -39,6 +39,8 @@ mod pg_executor;
mod php_executor;
#[cfg(feature = "python")]
mod python_executor;
#[cfg(feature = "python")]
mod python_versions;
pub mod result_processor;
#[cfg(feature = "rust")]
mod rust_executor;
@@ -60,3 +62,6 @@ pub use bun_executor::{
prebundle_bun_script, prepare_job_dir,
};
pub use deno_executor::generate_deno_lock;
#[cfg(feature = "python")]
pub use python_versions::{PyV, PyVAlias};
backend/windmill-worker/src/mssql_executor.rs
+1 -1
View File
@@ -22,7 +22,7 @@ use windmill_queue::{append_logs, CanceledBy};
use crate::common::{build_args_values, s3_mode_args_to_worker_data, OccupancyMetrics};
use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
use crate::AuthedClient;
use windmill_common::client::AuthedClient;
use serde::Deserializer;
backend/windmill-worker/src/mysql_executor.rs
+1 -1
View File
@@ -13,6 +13,7 @@ use serde_json::{json, value::RawValue, Value};
use std::str::FromStr;
use tokio::sync::Mutex;
use windmill_common::{
client::AuthedClient,
error::{to_anyhow, Error},
s3_helpers::convert_json_line_stream,
worker::{to_raw_value, Connection},
@@ -28,7 +29,6 @@ use crate::{
common::{build_args_values, s3_mode_args_to_worker_data, OccupancyMetrics, S3ModeWorkerData},
handle_child::run_future_with_polling_update_job_poller,
sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args,
AuthedClient,
};
#[derive(Deserialize)]
backend/windmill-worker/src/nu_executor.rs
+3 -1
View File
@@ -16,8 +16,10 @@ use crate::{
create_args_and_out_file, get_reserved_variables, read_result, start_child_process,
OccupancyMetrics,
},
handle_child, AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
handle_child, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
};
use windmill_common::client::AuthedClient;
const NSJAIL_CONFIG_RUN_NU_CONTENT: &str = include_str!("../nsjail/run.nu.config.proto");
lazy_static::lazy_static! {
backend/windmill-worker/src/oracledb_executor.rs
+2 -2
View File
@@ -27,9 +27,9 @@ use crate::{
OccupancyMetrics, S3ModeWorkerData,
},
handle_child::run_future_with_polling_update_job_poller,
sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args,
AuthedClient,
sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args
};
use windmill_common::client::AuthedClient;
#[derive(Deserialize)]
struct OracleDatabase {
backend/windmill-worker/src/pg_executor.rs
+2 -2
View File
@@ -41,11 +41,11 @@ use crate::common::{
};
use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
use crate::{AuthedClient, MAX_RESULT_SIZE};
use crate::MAX_RESULT_SIZE;
use bytes::Buf;
use lazy_static::lazy_static;
use urlencoding::encode;
use windmill_common::client::AuthedClient;
#[derive(Deserialize)]
struct PgDatabase {
host: String,
backend/windmill-worker/src/php_executor.rs
+2 -1
View File
@@ -20,9 +20,10 @@ use crate::{
read_result, start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
AuthedClient, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH,
COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH,
PHP_PATH,
};
use windmill_common::client::AuthedClient;
const NSJAIL_CONFIG_RUN_PHP_CONTENT: &str = include_str!("../nsjail/run.php.config.proto");
backend/windmill-worker/src/python_executor.rs
+151 -511
View File
@@ -3,6 +3,7 @@ use std::{
fs,
path::Path,
process::Stdio,
str::FromStr,
sync::Arc,
};
@@ -38,12 +39,12 @@ use std::env::var;
use windmill_queue::{append_logs, CanceledBy, PrecomputedAgentInfo};
lazy_static::lazy_static! {
static ref PYTHON_PATH: Option<String> = var("PYTHON_PATH").ok().map(|v| {
pub(crate) static ref PYTHON_PATH: Option<String> = var("PYTHON_PATH").ok().map(|v| {
tracing::warn!("PYTHON_PATH is set to {} and thus python will not be managed by uv and stay static regardless of annotation and instance settings. NOT RECOMMENDED", v);
v
});
static ref UV_PATH: String =
pub(crate) static ref UV_PATH: String =
var("UV_PATH").unwrap_or_else(|_| "/usr/local/bin/uv".to_string());
static ref PY_CONCURRENT_DOWNLOADS: usize =
@@ -69,7 +70,7 @@ const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
use crate::global_cache::{build_tar_and_push, pull_from_tar};
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
use windmill_common::s3_helpers::OBJECT_STORE_SETTINGS;
use crate::{
common::{
@@ -77,347 +78,11 @@ use crate::{
start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
worker_lockfiles::LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT,
worker_utils::ping_job_status,
AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, INSTANCE_PYTHON_VERSION, NSJAIL_PATH,
PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR,
PyV, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR,
};
// To change latest stable version:
// 1. Change placeholder in instanceSettings.ts
// 2. Change LATEST_STABLE_PY in dockerfile
// 3. Change #[default] annotation for PyVersion in backend
#[derive(Eq, PartialEq, Clone, Copy, Default, Debug)]
pub enum PyVersion {
Py310,
#[default]
Py311,
Py312,
Py313,
}
impl PyVersion {
pub async fn from_instance_version(job_id: &Uuid, w_id: &str, conn: &Connection) -> Self {
let mut err = None;
let pyv = match INSTANCE_PYTHON_VERSION.read().await.clone() {
Some(v) => PyVersion::from_string_with_dots(&v).unwrap_or_else(|| {
let v = PyVersion::default();
err = Some(format!("\nCannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to latest_stable ({v:?})", *INSTANCE_PYTHON_VERSION));
v
}),
// Use latest stable
None => PyVersion::default(),
};
if let Some(msg) = err {
append_logs(job_id, w_id, &msg, conn).await;
tracing::error!(msg);
}
pyv
}
/// e.g.: `/tmp/windmill/cache/python_3xy`
pub fn to_cache_dir(&self) -> String {
use windmill_common::worker::ROOT_CACHE_DIR;
format!("{ROOT_CACHE_DIR}{}", &self.to_cache_dir_top_level())
}
/// e.g.: `python_3xy`
pub fn to_cache_dir_top_level(&self) -> String {
format!("python_{}", self.to_string_no_dot())
}
/// e.g.: `3xy`
pub fn to_string_no_dot(&self) -> String {
self.to_string_with_dot().replace('.', "")
}
/// e.g.: `3.xy`
pub fn to_string_with_dot(&self) -> &str {
use PyVersion::*;
match self {
Py310 => "3.10",
Py311 => "3.11",
Py312 => "3.12",
Py313 => "3.13",
}
}
pub fn from_string_with_dots(value: &str) -> Option<Self> {
use PyVersion::*;
match value {
"3.10" => Some(Py310),
"3.11" => Some(Py311),
"3.12" => Some(Py312),
"3.13" => Some(Py313),
"default" => Some(PyVersion::default()),
_ => {
tracing::warn!(
"Cannot convert string (\"{value}\") to PyVersion\nExpected format x.yz"
);
None
}
}
}
pub fn from_string_no_dots(value: &str) -> Option<Self> {
use PyVersion::*;
match value {
"310" => Some(Py310),
"311" => Some(Py311),
"312" => Some(Py312),
"313" => Some(Py313),
"default" => Some(PyVersion::default()),
_ => {
tracing::warn!(
"Cannot convert string (\"{value}\") to PyVersion\nExpected format xyz"
);
None
}
}
}
/// e.g.: `# py3xy` -> `PyVersion::Py3XY`
pub fn parse_version(line: &str) -> Option<Self> {
Self::from_string_no_dots(line.replace(" ", "").replace("#py", "").as_str())
}
pub fn from_py_annotations(a: PythonAnnotations) -> Option<Self> {
let PythonAnnotations { py310, py311, py312, py313, .. } = a;
use PyVersion::*;
if py313 {
Some(Py313)
} else if py312 {
Some(Py312)
} else if py311 {
Some(Py311)
} else if py310 {
Some(Py310)
} else {
None
}
}
pub fn from_numeric(n: u32) -> Option<Self> {
use PyVersion::*;
match n {
310 => Some(Py310),
311 => Some(Py311),
312 => Some(Py312),
313 => Some(Py313),
_ => None,
}
}
pub fn to_numeric(&self) -> u32 {
use PyVersion::*;
match self {
Py310 => 310,
Py311 => 311,
Py312 => 312,
Py313 => 313,
}
}
pub async fn get_python(
&self,
job_id: &Uuid,
mem_peak: &mut i32,
// canceled_by: &mut Option<CanceledBy>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<Option<String>> {
// lazy_static::lazy_static! {
// static ref PYTHON_PATHS: Arc<RwLock<HashMap<PyVersion, String>>> = Arc::new(RwLock::new(HashMap::new()));
// }
let res = self
.get_python_inner(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics)
.await;
if let Err(ref e) = res {
tracing::error!(
"worker_name: {worker_name}, w_id: {w_id}, job_id: {job_id}\n
Error while getting python from uv, falling back to system python: {e:?}"
);
append_logs(
job_id,
w_id,
format!(
"\nError while getting python from uv, falling back to system python: {e:?}"
),
conn,
)
.await;
}
res
}
async fn get_python_inner(
self,
job_id: &Uuid,
mem_peak: &mut i32,
// canceled_by: &mut Option<CanceledBy>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<Option<String>> {
let py_path = self.find_python().await;
// Runtime is not installed
if py_path.is_err() {
// Install it
if let Err(err) = self
.install_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics)
.await
{
tracing::error!("Cannot install python: {err}");
return Err(err);
} else {
// Try to find one more time
let py_path = self.find_python().await;
if let Err(err) = py_path {
tracing::error!("Cannot find python version {err}");
return Err(err);
}
// TODO: Cache the result
py_path
}
} else {
py_path
}
}
async fn install_python(
self,
job_id: &Uuid,
mem_peak: &mut i32,
// canceled_by: &mut Option<CanceledBy>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<()> {
let v = self.to_string_with_dot();
append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), conn).await;
// Create dirs for newly installed python
// If we dont do this, NSJAIL will not be able to mount cache
// For the default version directory created during startup (main.rs)
DirBuilder::new()
.recursive(true)
.create(self.to_cache_dir())
.await
.expect("could not create initial worker dir");
let logs = String::new();
#[cfg(windows)]
let uv_cmd = "uv";
#[cfg(unix)]
let uv_cmd = UV_PATH.as_str();
let mut child_cmd = Command::new(uv_cmd);
child_cmd
.env_clear()
.env("HOME", HOME_ENV.to_string())
.env("PATH", PATH_ENV.to_string())
.envs(PROXY_ENVS.clone())
.args(["python", "install", v, "--python-preference=only-managed"])
// TODO: Do we need these?
.envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR)])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)]
{
child_cmd
.env("SystemRoot", SYSTEM_ROOT.as_str())
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
)
.env(
"LOCALAPPDATA",
std::env::var("LOCALAPPDATA")
.unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())),
);
}
let child_process = start_child_process(child_cmd, "uv").await?;
append_logs(&job_id, &w_id, logs, conn).await;
handle_child(
job_id,
conn,
mem_peak,
&mut None,
child_process,
false,
worker_name,
&w_id,
"uv",
None,
false,
occupancy_metrics,
None,
)
.await
}
async fn find_python(self) -> error::Result<Option<String>> {
#[cfg(windows)]
let uv_cmd = "uv";
#[cfg(unix)]
let uv_cmd = UV_PATH.as_str();
let mut child_cmd = Command::new(uv_cmd);
child_cmd.env_clear();
#[cfg(windows)]
{
child_cmd
.env("SystemRoot", SYSTEM_ROOT.as_str())
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
)
.env(
"LOCALAPPDATA",
std::env::var("LOCALAPPDATA")
.unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())),
);
}
let output = child_cmd
// .current_dir(job_dir)
.env("HOME", HOME_ENV.to_string())
.env("PATH", PATH_ENV.to_string())
.args([
"python",
"find",
self.to_string_with_dot(),
"--system",
"--python-preference=only-managed",
])
.envs([
("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR),
("UV_PYTHON_PREFERENCE", "only-managed"),
])
// .stdout(Stdio::piped())
.stderr(Stdio::piped())
.output()
.await?;
// Check if the command was successful
if output.status.success() {
// Convert the output to a String
let stdout =
String::from_utf8(output.stdout).expect("Failed to convert output to String");
return Ok(Some(stdout.replace('\n', "")));
} else {
// If the command failed, print the error
let stderr =
String::from_utf8(output.stderr).expect("Failed to convert error output to String");
return Err(error::Error::FindPythonError(stderr));
}
}
}
use windmill_common::client::AuthedClient;
#[cfg(windows)]
use crate::SYSTEM_ROOT;
@@ -465,7 +130,7 @@ pub async fn uv_pip_compile(
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
py_version: PyVersion,
py_version: PyV,
// Debug-only flag
no_cache: bool,
) -> error::Result<String> {
@@ -502,10 +167,11 @@ pub async fn uv_pip_compile(
requirements.to_string()
};
let py_version_str = py_version.clone().to_string();
// Include python version to requirements.in
// We need it because same hash based on requirements.in can get calculated even for different python versions
// To prevent from overwriting same requirements.in but with different python versions, we include version to hash
let requirements = format!("# py{}\n{}", py_version.to_string_no_dot(), requirements);
let requirements = format!("# py: {}\n{}", py_version.to_string(), requirements);
#[cfg(feature = "enterprise")]
let requirements = replace_pip_secret(conn, w_id, &requirements, worker_name, job_id).await?;
@@ -525,7 +191,7 @@ pub async fn uv_pip_compile(
{
logs.push_str(&format!(
"\nFound cached resolution: {req_hash}, on python version: {}",
py_version.to_string_with_dot()
&py_version_str
));
return Ok(cached);
}
@@ -539,7 +205,7 @@ pub async fn uv_pip_compile(
{
// Make sure we have python runtime installed
py_version
.get_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics)
.try_get_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics)
.await?;
let mut args = vec![
@@ -561,12 +227,7 @@ pub async fn uv_pip_compile(
UV_CACHE_DIR,
];
args.extend([
"-p",
&py_version.to_string_with_dot(),
"--python-preference",
"only-managed",
]);
args.extend(["-p", &py_version_str, "--python-preference", "only-managed"]);
if no_cache {
args.extend(["--no-cache"]);
@@ -666,8 +327,8 @@ pub async fn uv_pip_compile(
let mut req_content = "".to_string();
file.read_to_string(&mut req_content).await?;
let lockfile = format!(
"# py{}\n{}",
py_version.to_string_no_dot(),
"# py: {}\n{}",
py_version.to_string(),
req_content
.lines()
.filter(|x| !x.trim_start().starts_with('#'))
@@ -677,7 +338,7 @@ pub async fn uv_pip_compile(
);
if let Some(db) = conn.as_sql() {
sqlx::query!(
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('5 mins')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
req_hash,
lockfile
).fetch_optional(db).await?;
@@ -789,37 +450,6 @@ async fn postinstall(
Ok(())
}
async fn get_python_path(
py_version: PyVersion,
worker_name: &str,
job_id: &Uuid,
w_id: &str,
mem_peak: &mut i32,
conn: &Connection,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> windmill_common::error::Result<String> {
let python_path = if let Some(python_path) = PYTHON_PATH.clone() {
python_path
} else if let Some(python_path) = py_version
.get_python(
&job_id,
mem_peak,
conn,
worker_name,
w_id,
occupancy_metrics,
)
.await?
{
python_path
} else {
return Err(Error::ExecutionErr(format!(
"uv could not manage python path. Please manage it manually by setting PYTHON_PATH environment variable to your python binary path"
)));
};
Ok(python_path)
}
#[tracing::instrument(level = "trace", skip_all)]
pub async fn handle_python_job(
requirements_o: Option<&String>,
@@ -863,16 +493,16 @@ pub async fn handle_python_job(
.await?;
tracing::debug!("Finished handling python dependencies");
let python_path = get_python_path(
py_version,
worker_name,
&job.id,
&job.workspace_id,
mem_peak,
conn,
&mut Some(occupancy_metrics),
)
.await?;
let python_path = py_version
.get_python(
worker_name,
&job.id,
&job.workspace_id,
mem_peak,
conn,
&mut Some(occupancy_metrics),
)
.await?;
if !annotations.no_postinstall {
if let Err(e) = postinstall(&mut additional_python_paths, job_dir, job, conn).await {
@@ -887,7 +517,7 @@ pub async fn handle_python_job(
&job.workspace_id,
format!(
"\n\n--- PYTHON ({}) CODE EXECUTION ---\n",
py_version.to_string_with_dot()
py_version.clone().to_string()
),
conn,
)
@@ -1026,7 +656,7 @@ except BaseException as e:
let mut reserved_variables =
get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?;
// Add /tmp/windmill/cache/python_xyz/global-site-packages to PYTHONPATH.
// Add /tmp/windmill/cache/python_x_y_z/global-site-packages to PYTHONPATH.
// Usefull if certain wheels needs to be preinstalled before execution.
let global_site_packages_path = py_version.to_cache_dir() + "/global-site-packages";
let additional_python_paths_folders = {
@@ -1039,9 +669,9 @@ except BaseException as e:
// Since we handle mount of global_site_packages on our own, we don't want it to be mounted automatically.
// We do this because existence of every wheel in cache is mandatory and if it is not there and nsjail expects it, it is a bug.
// On the other side global_site_packages is purely optional.
// NOTE: This behaviour can be changed in future, so verification of wheels can be offloaded from nsjail to windmill
// NOTE: This behaviour can be changed in future, so verification of wheels can be delegated from nsjail to windmill
paths.insert(0, global_site_packages_path.clone());
// ^^^^^^^^
// ^^^^^^ ^
// We also want this be priorotized, that's why we insert it to the beginning
}
paths.iter().join(":")
@@ -1434,7 +1064,7 @@ async fn handle_python_deps(
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
precomputed_agent_info: Option<PrecomputedAgentInfo>,
annotations: PythonAnnotations,
) -> error::Result<(PyVersion, Vec<String>)> {
) -> error::Result<(PyV, Vec<String>)> {
create_dependencies_dir(job_dir).await;
let mut additional_python_paths: Vec<String> = WORKER_CONFIG
@@ -1445,90 +1075,116 @@ async fn handle_python_deps(
.unwrap_or_else(|| vec![])
.clone();
let mut requirements;
let compilation_error_hint;
let mut annotated_pyv = None;
let mut annotated_pyv_numeric = None;
let is_deployed = requirements_o.is_some();
let instance_pyv = PyVersion::from_instance_version(job_id, w_id, conn).await;
let requirements = match requirements_o {
Some(r) => r,
let (pyv, resolved_lines) = match requirements_o {
// Deployed
Some(r) => {
let rl = split_requirements(r);
(PyV::parse_from_requirements(&rl), rl)
}
// Preview
None => {
let mut already_visited = vec![];
(requirements, compilation_error_hint) = match conn {
let (v, requirements_lines, error_hint) = match conn {
Connection::Sql(db) => {
let mut version_specifiers = vec![];
let (r, h) = windmill_parser_py_imports::parse_python_imports(
inner_content,
w_id,
script_path,
db,
&mut already_visited,
&mut annotated_pyv_numeric,
&mut version_specifiers,
)
.await?;
(r.join("\n"), h)
let v = PyV::resolve(
version_specifiers,
job_id,
w_id,
annotations.py_select_latest,
Some(conn.clone()),
None,
None,
)
.await?;
(v, r, h)
}
Connection::Http(_) => match precomputed_agent_info {
Some(PrecomputedAgentInfo::Python { py_version, requirements }) => {
annotated_pyv_numeric = py_version;
(requirements.clone().unwrap_or_else(|| "".to_string()), None)
Some(PrecomputedAgentInfo::Python {
requirements,
py_version,
py_version_v2,
}) => {
let v = {
let v_v2 = py_version_v2
.clone()
.and_then(|s| pep440_rs::Version::from_str(&s).ok().map(PyV::from));
let v_v1 = py_version.and_then(PyVAlias::try_from_v1).map(PyV::from);
match v_v2.or(v_v1) {
Some(v) => v,
None => {
tracing::warn!(
workspace_id = %w_id,
"
Failed to get precomputed python version from server. Fallback to Default ({})
Returned from server: py_version - {:?}, py_version_v2 - {:?}
",
*PyV::default(),
py_version,
py_version_v2
);
Default::default()
}
}
};
let r = split_requirements(requirements.unwrap_or_default());
let h = None;
(v, r, h)
}
_ => ("".to_string(), None),
_ => Default::default(),
},
};
annotated_pyv = annotated_pyv_numeric.and_then(|v| PyVersion::from_numeric(v));
if !requirements.is_empty() {
requirements = uv_pip_compile(
job_id,
&requirements,
mem_peak,
canceled_by,
job_dir,
conn,
worker_name,
w_id,
occupancy_metrics,
annotated_pyv.unwrap_or(instance_pyv),
annotations.no_cache,
)
.await
.map_err(|e| {
Error::ExecutionErr(format!(
"pip compile failed: {}{}",
e.to_string(),
compilation_error_hint.unwrap_or_default()
))
})?;
}
&requirements
(
v.clone(),
if !requirements_lines.is_empty() {
uv_pip_compile(
job_id,
&requirements_lines.join("\n"),
mem_peak,
canceled_by,
job_dir,
conn,
worker_name,
w_id,
occupancy_metrics,
// annotated_pyv.unwrap_or(instance_pyv),
v,
annotations.no_cache,
)
.await
.map_err(|e| {
Error::ExecutionErr(format!(
"pip compile failed: {}{}",
e.to_string(),
error_hint.unwrap_or_default()
))
})?
.lines()
.map(|s| s.to_owned())
.collect_vec()
} else {
vec![]
},
)
}
};
/*
For deployed scripts we want to find out version in following order:
1. Assigned version (written in lockfile)
2. 3.11
For Previews:
1. Annotated version
2. Instance version
3. Latest Stable
*/
let requirements_lines = split_requirements(requirements.as_str());
let final_version = if is_deployed {
get_pyv_from_requirements_lines(&requirements_lines)
} else {
// This is not deployed script, meaning we test run it (Preview)
annotated_pyv.unwrap_or(instance_pyv)
};
// If len > 0 it means there is atleast one dependency or assigned python version
if requirements.len() > 0 {
if !resolved_lines.is_empty() {
let mut venv_path = handle_python_reqs(
requirements_lines,
resolved_lines,
job_id,
w_id,
mem_peak,
@@ -1538,13 +1194,13 @@ async fn handle_python_deps(
job_dir,
worker_dir,
occupancy_metrics,
final_version,
pyv.clone(),
)
.await?;
additional_python_paths.append(&mut venv_path);
}
Ok((final_version, additional_python_paths))
Ok((pyv, additional_python_paths))
}
lazy_static::lazy_static! {
@@ -1733,7 +1389,7 @@ async fn spawn_uv_install(
/// uv pip install, include cached or pull from S3
pub async fn handle_python_reqs(
requirements: Vec<&str>,
requirements: Vec<String>,
job_id: &Uuid,
w_id: &str,
mem_peak: &mut i32,
@@ -1743,7 +1399,7 @@ pub async fn handle_python_reqs(
job_dir: &str,
worker_dir: &str,
_occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
py_version: PyVersion,
py_version: PyV,
) -> error::Result<Vec<String>> {
let worker_dir = worker_dir.to_string();
@@ -1769,7 +1425,7 @@ pub async fn handle_python_reqs(
}
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
if OBJECT_STORE_CACHE_SETTINGS.read().await.is_none() {
if OBJECT_STORE_SETTINGS.read().await.is_none() {
(s3_pull, s3_push) = (false, false);
}
@@ -2017,7 +1673,7 @@ pub async fn handle_python_reqs(
let total_time = std::time::Instant::now();
let py_path = py_version
.get_python(
.try_get_python(
job_id,
mem_peak,
conn,
@@ -2059,6 +1715,10 @@ pub async fn handle_python_reqs(
let py_path = py_path.clone();
let pids = pids.clone();
let worker_dir = worker_dir.clone();
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
let py_version = py_version.clone();
handles.push(task::spawn(async move {
// permit will be dropped anyway if this thread exits at any point
// so we dont have to drop it manually
@@ -2076,7 +1736,7 @@ pub async fn handle_python_reqs(
let start = std::time::Instant::now();
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
if is_not_pro {
if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
tokio::select! {
// Cancel was called on the job
_ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")),
@@ -2230,7 +1890,7 @@ pub async fn handle_python_reqs(
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
if s3_push {
if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false));
}
}
@@ -2300,36 +1960,14 @@ pub async fn handle_python_reqs(
};
}
fn split_requirements(requirements: &str) -> Vec<&str> {
pub fn split_requirements<T: AsRef<str>>(requirements: T) -> Vec<String> {
requirements
.split("\n")
.as_ref()
.lines()
.filter(|x| !x.trim_start().starts_with("--") && !x.trim().is_empty())
.map(String::from)
.collect()
}
/// Check requirements/lockfile to figure out python version assigned to it.
fn get_pyv_from_requirements_lines(requirements_lines: &[&str]) -> PyVersion {
// If script is deployed we can try to parse first line to get assigned version
let index = if requirements_lines.get(0).map_or(false, |line| {
line.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT)
}) {
1
} else {
0
};
if let Some(v) = requirements_lines
.get(index)
.and_then(|line| PyVersion::parse_version(*line))
{
// We have valid assigned version, we use it
v
} else {
// If there is no assigned version in lockfile we automatically fallback to 3.11
// In this case we have dependencies, but no associated python version
// This is the case for old deployed scripts
PyVersion::Py311
}
}
// Returns code snippet that needs to be injected into wrapper to post-process results or leave unprocessed
fn get_result_postprocessor<'a>(skip: bool) -> &'a str {
@@ -2365,6 +2003,8 @@ pub async fn start_worker(
jobs_rx: tokio::sync::mpsc::Receiver<std::sync::Arc<MiniPulledJob>>,
killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> error::Result<()> {
use crate::{PyV, PyVAlias};
let mut mem_peak: i32 = 0;
let mut canceled_by: Option<CanceledBy> = None;
let context = variables::get_reserved_variables(
@@ -2518,22 +2158,22 @@ for line in sys.stdin:
proc_envs.insert("BASE_URL".to_string(), base_internal_url.to_string());
let py_version = if let Some(requirements) = requirements_o {
get_pyv_from_requirements_lines(&split_requirements(requirements.as_str()))
PyV::parse_from_requirements(&split_requirements(requirements.as_str()))
} else {
tracing::warn!(workspace_id = %w_id, "lockfile is empty for dedicated worker, thus python version cannot be inferred. Fallback to 3.11");
PyVersion::Py311
PyVAlias::Py311.into()
};
let python_path = get_python_path(
py_version,
worker_name,
&Uuid::nil(),
w_id,
&mut mem_peak,
&Connection::Sql(db.clone()),
&mut None,
)
.await?;
let python_path = py_version
.get_python(
worker_name,
&Uuid::nil(),
w_id,
&mut mem_peak,
&Connection::Sql(db.clone()),
&mut None,
)
.await?;
handle_dedicated_process(
&python_path,
job_dir,
backend/windmill-worker/src/python_versions.rs
+848
View File
@@ -0,0 +1,848 @@
use std::{
ops::{Deref, DerefMut},
process::Stdio,
str::FromStr,
sync::Arc,
};
use chrono::{DateTime, Duration, Utc};
use itertools::Itertools;
use serde_json::Value;
use tokio::{fs::DirBuilder, process::Command, sync::RwLock};
use uuid::Uuid;
use windmill_common::{
error::{self, Error},
worker::Connection,
};
use anyhow::{anyhow, bail};
use windmill_queue::append_logs;
use crate::{
common::{start_child_process, OccupancyMetrics},
handle_child::handle_child,
python_executor::{PYTHON_PATH, UV_PATH},
worker_lockfiles::LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT,
HOME_ENV, INSTANCE_PYTHON_VERSION, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, WIN_ENVS,
};
#[derive(Eq, PartialEq, Clone, Copy, Default, Debug)]
#[repr(u32)]
pub enum PyVAlias {
Py310 = 10,
#[default]
Py311,
Py312,
Py313,
}
impl Into<pep440_rs::Version> for PyVAlias {
fn into(self) -> pep440_rs::Version {
pep440_rs::Version::new([self.major() as u64, self as u64])
}
}
impl Into<u32> for PyVAlias {
fn into(self) -> u32 {
self.major() * 100 + self as u32
}
}
impl From<PyV> for PyVAlias {
fn from(value: PyV) -> Self {
match value.release() {
[major, minor, ..] => {
if let Some(alias) = Self::try_from_v1(format!("{}{}", *major, *minor)) {
return alias;
}
}
_ => (),
}
tracing::warn!(
"Failed to convert Python Full Version to Alias. Fallback to default ({})",
*PyV::default()
);
Self::default()
}
}
impl PyVAlias {
fn all<T: From<PyVAlias>>() -> Vec<T> {
use PyVAlias::*;
vec![Py310.into(), Py311.into(), Py312.into(), Py313.into()]
}
// Get MAJOR part of alias. (semver: MAJOR.MINOR.PATCH)
fn major(&self) -> u32 {
use PyVAlias::*;
match self {
Py310 | Py311 | Py312 | Py313 => 3,
// Py400 | Py401 => 4
}
}
/// Converts numeric format to alias
/// Example:
/// 310u32 (in) -> PyVAlias::Py310 (out)
pub(crate) fn try_from_v1<T: ToString>(numeric: T) -> Option<Self> {
use PyVAlias::*;
match numeric.to_string().as_str() {
"310" => Some(Py310),
"311" => Some(Py311),
"312" => Some(Py312),
"313" => Some(Py313),
_ => None,
}
}
}
// To change latest stable version:
// 1. Change placeholder in instanceSettings.ts
// 2. Change LATEST_STABLE_PY in dockerfile
// 3. Change #[default] annotation for PyVersion in backend
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct PyV(pub pep440_rs::Version);
impl From<pep440_rs::Version> for PyV {
fn from(value: pep440_rs::Version) -> Self {
Self(value)
}
}
impl From<PyVAlias> for PyV {
fn from(value: PyVAlias) -> Self {
Self(value.into())
}
}
impl Default for PyV {
fn default() -> Self {
PyVAlias::default().into()
}
}
impl Deref for PyV {
type Target = pep440_rs::Version;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl DerefMut for PyV {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
impl PyV {
pub async fn resolve(
version_specifiers: Vec<pep440_rs::VersionSpecifier>,
job_id: &Uuid,
w_id: &str,
select_latest: bool,
// Needed for logs but optional
conn: Option<Connection>,
// Usually for testing
custom_versions: Option<Vec<PyV>>,
// For testing
gravitational_version: Option<PyV>,
) -> Result<Self, Error> {
// Get all versions that can be fetched
let all_versions = custom_versions.unwrap_or(PyV::list_available_python_versions().await);
// Narrow down to those that satisfy given version specifiers
let valid = all_versions
.clone()
.into_iter()
.filter(|v| version_specifiers.iter().all(|vs| vs.contains(&*v)))
.collect_vec();
if !valid.is_empty() {
if select_latest {
return Ok(valid[0].clone());
}
// Usually INSTANCE_PYTHON_VERSION
let gv = gravitational_version
.unwrap_or(PyV::gravitational_version(job_id, w_id, conn).await);
// Will be used to determine if picked version matches gravity version
// Once first match occure, we will stop iterating
let gravity_matcher = pep440_rs::VersionSpecifier::from_version(
pep440_rs::Operator::EqualStar,
(*gv).clone(),
)
.map_err(|e| {
Error::ArgumentErr(format!(
"{e}\nLikely means INSTANCE_PYTHON_VERSION is set incorrectly."
))
})?;
// Reminder of semver: MAJOR.MINOR.PATCH
//
// - Go from up to down
// - We will iterate until find the closest version to target.
// - If closest version has the same MINOR version, use it.
// - If it differs in MINOR version, take latest PATCH version.
//
let mut result = None;
// This represents newest version with oldest MINOR:
//
// I Iterable Newest in MINOR
// 1. 3.11.2 -> 3.11.2
// 2. 3.11.1 -> 3.11.2
// 3. 3.11.0 -> 3.11.2
// 4. 3.10.2 -> 3.10.2
// 5. 3.10.1 -> 3.10.2
// 6. 3.10.0 -> 3.10.2
let mut newest_in_minor = None;
for v in valid.iter() {
if result.is_none() {
result.replace(v);
}
if v < &gv {
// We will not continue if we start looking into versions older than gravity version.
break;
}
let [major, minor, ..] = v.release() else {
return Err(Error::InternalErr(format!("Failed to parse \"{}\". Available python versions are supposed to be in SEMVER format (MAJOR.MINOR)", **v)));
};
// Since we go top to down we can assume
// the first occurence of new minor version contains the latest patch version.
if matches!(newest_in_minor, Some((_, mm)) if mm != (major, minor))
|| newest_in_minor.is_none()
{
newest_in_minor = Some((v.clone(), (major, minor)));
}
if gravity_matcher.contains(v) {
// return as soon as gravity matcher has first hit.
return Ok(v.clone());
}
// If we are still in the loop, it means that we are getting closer to gravity version
else {
result = Some(v);
}
}
let [gravity_major, gravity_minor, ..] = gv.release() else {
return Err(Error::internal_err(format!("Cannot get MAJOR or/and MINOR version of python gravity version ({}). Something might be wrong with INSTANCE_PYTHON_VERSION.", &*gv)));
};
if let Some((v, mm)) = newest_in_minor {
if (gravity_major, gravity_minor) != mm {
return Ok(v);
}
}
result
.ok_or(Error::internal_err(
"No python candidates found. This is a bug!",
))
.map(ToOwned::to_owned)
} else {
Err(anyhow!(
"
× No solution found when resolving python:
╰─▶ Because you require python {}, we can conclude that your requirements are unsatisfiable.
All versions: \n{}
\n",
version_specifiers.iter().map(|s| s.to_string()).join(", "),
all_versions
.iter()
.enumerate()
.map(|(i, v)| format!(
"{}{}",
windmill_common::worker::pad_string(&v.0.to_string(), 11),
if (i + 1) % 5 == 0 { "\n" } else { "" }
))
.collect::<String>()
)
.into())
}
}
/// e.g.: `/tmp/windmill/cache/python_3xy`
pub(crate) fn to_cache_dir(&self) -> String {
use windmill_common::worker::ROOT_CACHE_DIR;
format!("{ROOT_CACHE_DIR}{}", self.to_cache_dir_top_level())
}
/// e.g.: `python_3_x_y`
pub fn to_cache_dir_top_level(&self) -> String {
format!("python_{}", self.to_string().replace(".", "_"))
}
pub async fn gravitational_version(
job_id: &Uuid,
w_id: &str,
conn: Option<Connection>,
) -> Self {
let mut err = None;
let pyv = match INSTANCE_PYTHON_VERSION.read().await.clone() {
Some(v) => pep440_rs::Version::from_str(&v).unwrap_or_else(|_| {
let v = PyVAlias::default().into();
err = Some(format!("\nCannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to latest_stable ({v:?})", *INSTANCE_PYTHON_VERSION));
v
}),
// Use latest stable
None => PyVAlias::default().into(),
};
if let Some(msg) = err {
if let Some(conn) = conn {
append_logs(job_id, w_id, &msg, &conn).await;
}
tracing::error!(msg);
}
pyv.into()
}
pub async fn list_available_python_versions() -> Vec<Self> {
match Self::list_available_python_versions_inner().await {
Ok(pyvs) => pyvs,
Err(e) => {
tracing::error!(
"Fallback to preconfigured aliases. Cannot list python versions due to this error: {e}"
);
PyVAlias::all()
}
}
}
async fn list_available_python_versions_inner() -> anyhow::Result<Vec<Self>> {
lazy_static::lazy_static! {
static ref CACHED_VERSIONS: Arc<RwLock<Option<Vec<PyV>>>> = Arc::new(RwLock::new(None));
static ref LAST_CHECKED: Arc<RwLock<DateTime<Utc>>> = Arc::new(RwLock::new(Utc::now()));
}
match (
Utc::now().signed_duration_since(*LAST_CHECKED.read().await) > Duration::minutes(30),
CACHED_VERSIONS.read().await.clone(),
) {
(false, Some(vs)) => return Ok(vs),
_ => {}
};
let output = {
#[cfg(windows)]
let uv_cmd = "uv";
#[cfg(unix)]
let uv_cmd = UV_PATH.as_str();
Command::new(uv_cmd)
.env_clear()
.envs(WIN_ENVS.to_vec())
.args([
"python",
"list",
"--all-versions",
"--output-format",
"json",
])
.stderr(Stdio::piped())
.output()
.await?
};
// We want to skip all versions smaller then 3.10
// Windmill is incompatible with 3.9 and older
let filter = pep440_rs::VersionSpecifier::from_version(
pep440_rs::Operator::GreaterThanEqual,
PyVAlias::Py310.into(),
)?;
if output.status.success() {
let res = String::from_utf8(output.stdout)?;
tracing::error!("{}", &res);
let list = serde_json::from_str::<Vec<serde_json::Map<String, Value>>>(&res)?
.into_iter()
.filter_map(|e| {
if e.get("implementation").and_then(Value::as_str) == Some("pypy") {
None
} else {
Some(
e.get("version")
.and_then(Value::as_str)
.and_then(|s| pep440_rs::Version::from_str(s).ok())
.map(PyV::from)
.ok_or(Error::internal_err("version is None")),
)
}
})
.collect::<Result<Vec<PyV>, Error>>()?
.into_iter()
.unique()
.sorted()
.filter(|pyv| filter.contains(&*pyv))
.rev()
.collect_vec();
*LAST_CHECKED.write().await = Utc::now();
CACHED_VERSIONS.write().await.replace(list.clone());
Ok(list)
} else {
// If the command failed, print the error
let stderr = String::from_utf8(output.stderr)?;
bail!(
"Cannot list python versions, is uv (0.5.19 and newer) installed? Err:\n{}",
stderr
);
}
}
/// Parse lockfile for assigned python version.
/// If not found returns 3.11
pub fn parse_from_requirements<S: AsRef<str>>(requirements_lines: &[S]) -> Self {
Self::try_parse_from_requirements(requirements_lines).unwrap_or(
// If there is no assigned version in lockfile we automatically fallback to 3.11
// In this case we have dependencies or other metadata, but no associated python version
// This is the case for old deployed scripts
PyVAlias::Py311.into(),
)
}
/// Parse lockfile for assigned python version.
/// If not found returns None
pub fn try_parse_from_requirements<S: AsRef<str>>(requirements_lines: &[S]) -> Option<Self> {
let parse_version = |s: &str| -> Option<PyV> {
// Possible inputs:
// V2:
// # py: 3.11.0 or #py:3.11.0 or #py: 3.11.0
//
// V1:
// # py311 or #py311
let version_unparsed = s
.to_owned()
// Remove whitespaces. That leaves us with:
// V2: #py:3.11.0
// V1: #py311
//
// Remove #
// V2: py:3.11.0
// V1: py311
//
// Remove :
// V2: py3.11.0
// V1: py311
.replace([' ', '#', ':'], "")
// Remove "py"
// V2: 3.11.0
// V1: 311
.replace("py", "");
// We will support reading V1 syntax, but it will be overwritten next deploy
PyVAlias::try_from_v1(&version_unparsed)
.map(PyVAlias::into)
.or(pep440_rs::Version::from_str(&version_unparsed)
.ok()
.map(pep440_rs::Version::into))
};
let index = if requirements_lines.get(0).map_or(false, |line| {
line.as_ref()
.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT)
}) {
1
} else {
0
};
requirements_lines
.get(index)
.map(S::as_ref)
.and_then(parse_version)
}
pub async fn get_python(
&self,
worker_name: &str,
job_id: &Uuid,
w_id: &str,
mem_peak: &mut i32,
conn: &Connection,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> windmill_common::error::Result<String> {
let python_path = if let Some(python_path) = PYTHON_PATH.clone() {
python_path
} else if let Some(python_path) = self
.try_get_python(
&job_id,
mem_peak,
conn,
worker_name,
w_id,
occupancy_metrics,
)
.await?
{
python_path
} else {
return Err(Error::ExecutionErr(format!(
"uv could not manage python path. Please manage it manually by setting PYTHON_PATH environment variable to your python binary path"
)));
};
Ok(python_path)
}
pub async fn try_get_python(
&self,
job_id: &Uuid,
mem_peak: &mut i32,
// canceled_by: &mut Option<CanceledBy>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<Option<String>> {
// lazy_static::lazy_static! {
// static ref PYTHON_PATHS: Arc<RwLock<HashMap<PyVersion, String>>> = Arc::new(RwLock::new(HashMap::new()));
// }
let res = self
.get_python_inner(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics)
.await;
if let Err(ref e) = res {
tracing::error!(
"worker_name: {worker_name}, w_id: {w_id}, job_id: {job_id}\n
Error while getting python from uv, falling back to system python: {e:?}"
);
append_logs(
job_id,
w_id,
format!(
"\nError while getting python from uv, falling back to system python: {e:?}"
),
conn,
)
.await;
}
res
}
async fn get_python_inner(
&self,
job_id: &Uuid,
mem_peak: &mut i32,
// canceled_by: &mut Option<CanceledBy>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<Option<String>> {
let py_path = self.find_python().await;
// Runtime is not installed
if py_path.is_err() {
// Install it
if let Err(err) = self
.install_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics)
.await
{
tracing::error!("Cannot install python: {err}");
return Err(err);
} else {
// Try to find one more time
let py_path = self.find_python().await;
if let Err(err) = py_path {
tracing::error!("Cannot find python version {err}");
return Err(err);
}
// TODO: Cache the result
py_path
}
} else {
py_path
}
}
async fn install_python(
&self,
job_id: &Uuid,
mem_peak: &mut i32,
// canceled_by: &mut Option<CanceledBy>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<()> {
let v = self.to_string();
append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), conn).await;
// Create dirs for newly installed python
// If we dont do this, NSJAIL will not be able to mount cache
// For the default version directory created during startup (main.rs)
DirBuilder::new()
.recursive(true)
.create(self.to_cache_dir())
.await
.expect("could not create initial worker dir");
let logs = String::new();
#[cfg(windows)]
let uv_cmd = "uv";
#[cfg(unix)]
let uv_cmd = UV_PATH.as_str();
let mut child_cmd = Command::new(uv_cmd);
child_cmd
.env_clear()
.env("HOME", HOME_ENV.to_string())
.env("PATH", PATH_ENV.to_string())
.envs(PROXY_ENVS.clone())
.args(["python", "install", &v, "--python-preference=only-managed"])
// TODO: Do we need these?
.envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR)])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)]
{
child_cmd
.env("SystemRoot", crate::SYSTEM_ROOT.as_str())
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
)
.env(
"LOCALAPPDATA",
std::env::var("LOCALAPPDATA")
.unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())),
);
}
let child_process = start_child_process(child_cmd, "uv").await?;
append_logs(&job_id, &w_id, logs, conn).await;
handle_child(
job_id,
conn,
mem_peak,
&mut None,
child_process,
false,
worker_name,
&w_id,
"uv",
None,
false,
occupancy_metrics,
None,
)
.await
}
async fn find_python(&self) -> error::Result<Option<String>> {
#[cfg(windows)]
let uv_cmd = "uv";
#[cfg(unix)]
let uv_cmd = UV_PATH.as_str();
let mut child_cmd = Command::new(uv_cmd);
child_cmd.env_clear();
#[cfg(windows)]
{
child_cmd
.env("SystemRoot", crate::SYSTEM_ROOT.as_str())
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
)
.env(
"LOCALAPPDATA",
std::env::var("LOCALAPPDATA")
.unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())),
);
}
let output = child_cmd
// .current_dir(job_dir)
.env("HOME", HOME_ENV.to_string())
.env("PATH", PATH_ENV.to_string())
.args([
"python",
"find",
&self.to_string(),
"--system",
"--python-preference=only-managed",
])
.envs([
("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR),
("UV_PYTHON_PREFERENCE", "only-managed"),
])
// .stdout(Stdio::piped())
.stderr(Stdio::piped())
.output()
.await?;
// Check if the command was successful
if output.status.success() {
// Convert the output to a String
let stdout =
String::from_utf8(output.stdout).expect("Failed to convert output to String");
return Ok(Some(stdout.replace('\n', "")));
} else {
// If the command failed, print the error
let stderr =
String::from_utf8(output.stderr).expect("Failed to convert error output to String");
return Err(error::Error::FindPythonError(stderr));
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Unsafe helper for testing
fn pyv(value: &str) -> PyV {
pep440_rs::Version::from_str(value).unwrap().into()
}
async fn assert_resolution(
instance_version: &str,
select_highest: bool,
specifiers: Vec<&str>,
available: Vec<PyV>,
expected: PyV,
) {
let resolved = PyV::resolve(
specifiers
.into_iter()
.map(|s| pep440_rs::VersionSpecifier::from_str(s).unwrap())
.collect_vec(),
&Uuid::nil(),
"",
select_highest,
None,
Some(available),
Some(pyv(instance_version)),
)
.await
.unwrap();
assert_eq!(expected, resolved);
}
#[tokio::test]
async fn test_python_resolution_1() {
assert_resolution(
"1.0",
false,
vec![],
vec![
pyv("1.2.0"),
pyv("1.1.0"),
pyv("1.0.0"),
pyv("0.9.0"), //
],
pyv("1.0.0"), //
)
.await;
}
#[tokio::test]
async fn test_python_resolution_2() {
assert_resolution(
"1.0.0",
false,
vec!["!=1.*"],
vec![
pyv("1.2"),
pyv("1.1"),
pyv("1.0.2"),
pyv("1.0.1"),
pyv("1.0.0"),
pyv("0.9.4"),
pyv("0.9.3"),
pyv("0.9.2"),
],
pyv("0.9.4"), //
)
.await;
}
#[tokio::test]
async fn test_python_resolution_3() {
assert_resolution(
"0.9",
false,
vec!["!=0.9.*"],
vec![
pyv("1.2"),
pyv("1.1"),
pyv("1.0.2"),
pyv("1.0.1"),
pyv("1.0.0"),
pyv("0.9.4"),
pyv("0.9.3"),
pyv("0.9.2"),
pyv("0.8.2"),
pyv("0.8.1"),
pyv("0.8.0"),
],
pyv("1.0.2"), //
)
.await;
}
#[tokio::test]
async fn test_python_resolution_4() {
assert_resolution(
"0.9",
false,
vec!["<=0.8.1"],
vec![pyv("1.0.0"), pyv("0.9.0"), pyv("0.8.1"), pyv("0.8.0")],
pyv("0.8.1"), //
)
.await;
}
#[tokio::test]
async fn test_python_resolution_5() {
assert_resolution(
"0.0.1",
false,
vec!["!=0.1.0"],
vec![pyv("2.1.0"), pyv("1.1.0"), pyv("0.1.0")],
pyv("1.1.0"),
)
.await;
}
#[tokio::test]
async fn test_python_resolution_6() {
assert_resolution(
"1.1.1",
false,
vec![],
vec![
pyv("3.0.1"),
pyv("3.0.0"),
pyv("2.2.2"),
pyv("2.2.1"),
pyv("2.2.0"),
],
pyv("2.2.2"),
)
.await;
}
#[tokio::test]
async fn test_python_resolution_7() {
assert_resolution(
"2.2.1",
true,
vec![],
vec![
pyv("3.0.1"),
pyv("3.0.0"),
pyv("2.2.2"),
pyv("2.2.1"),
pyv("2.2.0"),
],
pyv("3.0.1"),
)
.await;
}
}
backend/windmill-worker/src/result_processor.rs
+5 -10
View File
@@ -29,8 +29,7 @@ use windmill_common::{
use windmill_common::bench::{BenchmarkInfo, BenchmarkIter};
use windmill_queue::{
append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob,
WrappedError,
append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob, WrappedError,
};
use serde_json::{json, value::RawValue};
@@ -44,9 +43,10 @@ use crate::{
common::{error_to_value, read_result, save_in_cache},
otel_ee::add_root_flow_job_to_otlp,
worker_flow::update_flow_status_after_job_completion,
AuthedClient, JobCompletedReceiver, JobCompletedSender, SameWorkerSender, SendResult,
UpdateFlow, INIT_SCRIPT_TAG,
JobCompletedReceiver, JobCompletedSender, SameWorkerSender, SendResult, UpdateFlow,
INIT_SCRIPT_TAG,
};
use windmill_common::client::AuthedClient;
async fn process_jc(
jc: JobCompleted,
@@ -273,11 +273,7 @@ pub fn start_background_processor(
})
}
async fn send_job_completed(
job_completed_tx: JobCompletedSender,
jc: JobCompleted,
) {
async fn send_job_completed(job_completed_tx: JobCompletedSender, jc: JobCompleted) {
job_completed_tx
.send_job(jc, true)
.with_context(windmill_common::otel_ee::otel_ctx())
@@ -301,7 +297,6 @@ pub async fn process_result(
) -> error::Result<bool> {
match result {
Ok(result) => {
send_job_completed(
job_completed_tx,
JobCompleted {
backend/windmill-worker/src/rust_executor.rs
+2 -1
View File
@@ -19,9 +19,10 @@ use crate::{
read_result, start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
RUST_CACHE_DIR, TZ_ENV,
};
use windmill_common::client::AuthedClient;
#[cfg(windows)]
use crate::SYSTEM_ROOT;
backend/windmill-worker/src/snowflake_executor.rs
+2 -1
View File
@@ -26,7 +26,8 @@ use crate::common::{
};
use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
use crate::{common::build_args_values, AuthedClient};
use crate::common::build_args_values;
use windmill_common::client::AuthedClient;
#[derive(Serialize)]
struct Claims {
backend/windmill-worker/src/worker.rs
+27 -203
View File
@@ -11,6 +11,7 @@
use anyhow::anyhow;
use futures::TryFutureExt;
use windmill_common::client::AuthedClient;
use windmill_common::{
agent_workers::DECODED_AGENT_TOKEN,
apps::AppScriptId,
@@ -28,7 +29,7 @@ use windmill_common::{
#[cfg(feature = "enterprise")]
use windmill_common::ee::LICENSE_KEY_VALID;
use anyhow::{Context, Result};
use anyhow::Result;
use const_format::concatcp;
#[cfg(feature = "prometheus")]
use prometheus::IntCounter;
@@ -39,8 +40,7 @@ use windmill_common::METRICS_DEBUG_ENABLED;
#[cfg(feature = "prometheus")]
use windmill_common::METRICS_ENABLED;
use reqwest::{Body, Response};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use serde::{Deserialize, Serialize};
use sqlx::types::Json;
use std::{
collections::HashMap,
@@ -134,7 +134,10 @@ use crate::java_executor::{handle_java_job, JobHandlerInput as JobHandlerInputJa
use crate::php_executor::handle_php_job;
#[cfg(feature = "python")]
use crate::python_executor::{handle_python_job, PyVersion};
use crate::{
python_executor::handle_python_job,
python_versions::{PyV, PyVAlias},
};
#[cfg(feature = "python")]
use crate::ansible_executor::handle_ansible_job;
@@ -363,10 +366,26 @@ lazy_static::lazy_static! {
}
type Envs = Vec<(String, String)>;
#[cfg(windows)]
lazy_static::lazy_static! {
pub static ref SYSTEM_ROOT: String = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string());
pub static ref USERPROFILE_ENV: String = std::env::var("USERPROFILE").unwrap_or_else(|_| "/tmp".to_string());
static ref TMP: String = std::env::var("TMP").unwrap_or_else(|_| "/tmp".to_string());
static ref LOCALAPPDATA: String = std::env::var("LOCALAPPDATA").unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str()));
pub static ref WIN_ENVS: Envs = vec![
("SystemRoot".into(), SYSTEM_ROOT.clone()),
("USERPROFILE".into(), USERPROFILE_ENV.clone()),
("TMP".into(), TMP.clone()),
("LOCALAPPDATA".into(), LOCALAPPDATA.clone())
];
}
#[cfg(not(windows))]
lazy_static::lazy_static! {
pub static ref WIN_ENVS: Envs = vec![];
}
//only matter if CLOUD_HOSTED
@@ -374,201 +393,6 @@ pub const MAX_RESULT_SIZE: usize = 1024 * 1024 * 2; // 2MB
pub const INIT_SCRIPT_TAG: &str = "init_script";
#[derive(Clone)]
pub struct AuthedClient {
pub base_internal_url: String,
pub workspace: String,
pub token: String,
pub force_client: Option<reqwest::Client>,
}
impl AuthedClient {
pub async fn get(&self, url: &str, query: Vec<(&str, String)>) -> anyhow::Result<Response> {
self.force_client
.as_ref()
.unwrap_or(&HTTP_CLIENT)
.get(url)
.query(&query)
.header(
reqwest::header::ACCEPT,
reqwest::header::HeaderValue::from_static("application/json"),
)
.header(
reqwest::header::AUTHORIZATION,
reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))?,
)
.send()
.await
.map_err(|e| {
tracing::error!("Error executing get request from authed http client to {url} with query {query:?}: {e}");
anyhow::anyhow!("Error executing get request from authed http client to {url} with query {query:?}: {e}")
})
}
pub async fn get_id_token(&self, audience: &str) -> anyhow::Result<String> {
let url = format!(
"{}/api/w/{}/oidc/token/{}",
self.base_internal_url, self.workspace, audience
);
let response = self.get(&url, vec![]).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<String>()
.await
.context("decoding oidc token as json string")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_resource_value<T: DeserializeOwned>(&self, path: &str) -> anyhow::Result<T> {
let url = format!(
"{}/api/w/{}/resources/get_value/{}",
self.base_internal_url, self.workspace, path
);
let response = self.get(&url, vec![]).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<T>()
.await
.context("decoding resource value as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_variable_value(&self, path: &str) -> anyhow::Result<String> {
let url = format!(
"{}/api/w/{}/variables/get_value/{}",
self.base_internal_url, self.workspace, path
);
let response = self.get(&url, vec![]).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<String>()
.await
.context("decoding variable value as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_resource_value_interpolated<T: DeserializeOwned>(
&self,
path: &str,
job_id: Option<String>,
) -> anyhow::Result<T> {
let url = format!(
"{}/api/w/{}/resources/get_value_interpolated/{}",
self.base_internal_url, self.workspace, path
);
let mut query = Vec::with_capacity(1usize);
if let Some(v) = &job_id {
query.push(("job_id", v.to_string()));
}
let response = self.get(&url, query).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<T>()
.await
.context("decoding interpolated resource value as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_completed_job_result<T: DeserializeOwned>(
&self,
path: &str,
json_path: Option<String>,
) -> anyhow::Result<T> {
let url = format!(
"{}/api/w/{}/jobs_u/completed/get_result/{}",
self.base_internal_url, self.workspace, path
);
let query = if let Some(json_path) = json_path {
vec![("json_path", json_path)]
} else {
vec![]
};
let response = self.get(&url, query).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<T>()
.await
.context("decoding completed job result as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_result_by_id<T: DeserializeOwned>(
&self,
flow_job_id: &str,
node_id: &str,
json_path: Option<String>,
) -> anyhow::Result<T> {
let url = format!(
"{}/api/w/{}/jobs/result_by_id/{}/{}",
self.base_internal_url, self.workspace, flow_job_id, node_id
);
let query = if let Some(json_path) = json_path {
vec![("json_path", json_path)]
} else {
vec![]
};
let response = self.get(&url, query).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<T>()
.await
.context("decoding result by id as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn upload_s3_file<S>(
&self,
workspace_id: &str,
object_key: String,
storage: Option<String>,
body: S,
) -> error::Result<()>
where
S: futures::stream::TryStream + Send + 'static,
S::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
bytes::Bytes: From<S::Ok>,
{
let mut query = vec![("file_key", object_key)];
if let Some(storage) = storage {
query.push(("storage", storage));
}
let response = self
.force_client
.as_ref()
.unwrap_or(&HTTP_CLIENT)
.post(format!(
"{}/api/w/{}/job_helpers/upload_s3_file",
self.base_internal_url, workspace_id
))
.query(&query)
.header(
reqwest::header::ACCEPT,
reqwest::header::HeaderValue::from_static("application/json"),
)
.header(
reqwest::header::AUTHORIZATION,
reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))
.map_err(|e| error::Error::BadConfig(e.to_string()))?,
)
.body(Body::wrap_stream(body))
.send()
.await
.context(format!("Sent upload_s3_file request",))
.map_err(error::Error::from)?;
match response.status().as_u16() {
200u16 => Ok(()),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?,
}
}
}
#[derive(Clone)]
pub struct SameWorkerSender(pub Sender<SameWorkerPayload>, pub Arc<AtomicU16>);
@@ -828,9 +652,9 @@ pub async fn run_worker(
worker_dir.clone(),
);
tokio::spawn(async move {
if let Err(e) = PyVersion::from_instance_version(&Uuid::nil(), "", &conn)
if let Err(e) = PyV::gravitational_version(&Uuid::nil(), "", Some(conn.clone()))
.await
.get_python(&Uuid::nil(), &mut 0, &conn, &worker_name, "", &mut None)
.try_get_python(&Uuid::nil(), &mut 0, &conn, &worker_name, "", &mut None)
.await
{
tracing::error!(
@@ -840,8 +664,8 @@ pub async fn run_worker(
"Cannot preinstall or find Instance Python version to worker: {e}"//
);
}
if let Err(e) = PyVersion::Py311
.get_python(&Uuid::nil(), &mut 0, &conn, &worker_name, "", &mut None)
if let Err(e) = PyV::from(PyVAlias::Py311)
.try_get_python(&Uuid::nil(), &mut 0, &conn, &worker_name, "", &mut None)
.await
{
tracing::error!(
backend/windmill-worker/src/worker_flow.rs
+2 -2
View File
@@ -15,8 +15,7 @@ use crate::common::{cached_result_path, save_in_cache};
use crate::js_eval::{eval_timeout, IdContext};
use crate::worker_utils::get_tag_and_concurrency;
use crate::{
AuthedClient, JobCompletedSender, PreviousResult, SameWorkerSender, SendResult, UpdateFlow,
KEEP_JOB_DIR,
JobCompletedSender, PreviousResult, SameWorkerSender, SendResult, UpdateFlow, KEEP_JOB_DIR,
};
use anyhow::Context;
use futures::TryFutureExt;
@@ -32,6 +31,7 @@ use windmill_common::auth::JobPerms;
#[cfg(feature = "benchmark")]
use windmill_common::bench::BenchmarkIter;
use windmill_common::cache::{self, RawData};
use windmill_common::client::AuthedClient;
use windmill_common::db::Authed;
use windmill_common::flow_status::{
ApprovalConditions, FlowStatusModuleWParent, Iterator as FlowIterator, JobResult,
backend/windmill-worker/src/worker_lockfiles.rs
+48 -47
View File
@@ -47,7 +47,7 @@ use crate::java_executor::resolve;
use crate::php_executor::{composer_install, parse_php_imports};
#[cfg(feature = "python")]
use crate::python_executor::{
create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion,
create_dependencies_dir, handle_python_reqs, split_requirements, uv_pip_compile,
};
#[cfg(feature = "rust")]
use crate::rust_executor::generate_cargo_lockfile;
@@ -1897,26 +1897,13 @@ async fn python_dep(
w_id: &str,
worker_dir: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
annotated_pyv_numeric: Option<u32>,
py_version: crate::PyV,
annotations: PythonAnnotations,
) -> std::result::Result<String, Error> {
use crate::python_executor::split_requirements;
create_dependencies_dir(job_dir).await;
/*
Unlike `handle_python_deps` which we use for running scripts (deployed and drafts)
This one used specifically for deploying scripts
So we can get final_version right away and include in lockfile
And the precendence is following:
1. Annotation version
2. Instance version
3. Latest Stable
*/
let final_version = annotated_pyv_numeric
.and_then(|pyv| PyVersion::from_numeric(pyv))
.unwrap_or(PyVersion::from_instance_version(job_id, w_id, &db.into()).await);
let req: std::result::Result<String, Error> = uv_pip_compile(
job_id,
&reqs,
@@ -1927,14 +1914,15 @@ async fn python_dep(
worker_name,
w_id,
occupancy_metrics,
final_version,
py_version,
annotations.no_cache,
)
.await;
// install the dependencies to pre-fill the cache
if let Ok(req) = req.as_ref() {
let r = handle_python_reqs(
req.split("\n").filter(|x| !x.starts_with("--")).collect(),
split_requirements(req),
// req.split("\n").filter(|x| !x.starts_with("--")).collect(),
job_id,
w_id,
mem_peak,
@@ -1944,7 +1932,8 @@ async fn python_dep(
job_dir,
worker_dir,
occupancy_metrics,
final_version,
// final_version,
crate::PyVAlias::default().into(),
)
.await;
@@ -1975,13 +1964,12 @@ async fn ansible_dep(
) -> std::result::Result<String, Error> {
use windmill_parser_yaml::add_versions_to_requirements_yaml;
use crate::{
ansible_executor::{
use crate::ansible_executor::{
create_ansible_cfg, get_collection_locks, get_git_ssh_cmd, get_role_locks,
install_galaxy_collections,
},
AuthedClient,
};
};
use windmill_common::client::AuthedClient;
let python_lockfile = python_dep(
reqs.python_reqs.join("\n").to_string(),
@@ -1994,7 +1982,7 @@ async fn ansible_dep(
w_id,
worker_dir,
&mut Some(occupancy_metrics),
None,
crate::PyV::gravitational_version(job_id, w_id, Some(db.clone().into())).await,
PythonAnnotations::default(),
)
.await?;
@@ -2104,31 +2092,44 @@ async fn capture_dependency_job(
));
#[cfg(feature = "python")]
{
let anns = PythonAnnotations::parse(job_raw_code);
let mut annotated_pyv_numeric = None;
let reqs = if raw_deps {
// Manually assigned version from requirements.txt
// let assigned_py_version;
let (reqs, py_version) = if raw_deps {
// `wmill script generate-metadata`
// should also respect annotated pyversion
// can be annotated in script itself
// or in requirements.txt if present
annotated_pyv_numeric =
PyVersion::from_py_annotations(anns).map(|v| v.to_numeric());
job_raw_code.to_string()
} else {
let mut already_visited = vec![];
windmill_parser_py_imports::parse_python_imports(
job_raw_code,
&w_id,
script_path,
&db,
&mut already_visited,
&mut annotated_pyv_numeric,
(
job_raw_code.to_owned(),
crate::PyV::parse_from_requirements(&split_requirements(job_raw_code)),
)
} else {
let mut version_specifiers = vec![];
let PythonAnnotations { py_select_latest, .. } =
PythonAnnotations::parse(job_raw_code);
(
windmill_parser_py_imports::parse_python_imports(
job_raw_code,
&w_id,
script_path,
&db,
&mut version_specifiers,
)
.await?
.0
.join("\n"),
crate::PyV::resolve(
version_specifiers,
job_id,
w_id,
py_select_latest,
Some(db.clone().into()),
None,
None,
)
.await?,
)
.await?
.0
.join("\n")
};
python_dep(
@@ -2142,8 +2143,8 @@ async fn capture_dependency_job(
w_id,
worker_dir,
&mut Some(occupancy_metrics),
annotated_pyv_numeric,
anns,
py_version,
PythonAnnotations::parse(job_raw_code),
)
.await
.map(|res| {
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.491.5";
export const VERSION = "v1.492.1";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
cli/main.ts
+1 -1
View File
@@ -63,7 +63,7 @@ export {
// }
// });
export const VERSION = "1.491.5";
export const VERSION = "1.492.1";
const command = new Command()
.name("wmill")
frontend/package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "windmill-components",
"version": "1.491.5",
"version": "1.492.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-components",
"version": "1.491.5",
"version": "1.492.1",
"hasInstallScript": true,
"license": "AGPL-3.0",
"dependencies": {
frontend/package.json
+8 -1
View File
@@ -1,6 +1,6 @@
{
"name": "windmill-components",
"version": "1.491.5",
"version": "1.492.1",
"scripts": {
"dev": "vite dev",
"build": "vite build",
@@ -372,6 +372,10 @@
"types": "./package/components/SimpleEditor.svelte.d.ts",
"svelte": "./package/components/SimpleEditor.svelte",
"default": "./package/components/SimpleEditor.svelte"
},
"./tailwindUtils": {
"types": "./package/components/apps/editor/componentsPanel/tailwindUtils.d.ts",
"default": "./package/components/apps/editor/componentsPanel/tailwindUtils.js"
}
},
"files": [
@@ -507,6 +511,9 @@
],
"components/icons/store": [
"./package/components/icons/store.d.ts"
],
"tailwindUtils": [
"./package/components/apps/editor/componentsPanel/tailwindUtils.d.ts"
]
}
},
frontend/src/lib/components/FlowBuilder.svelte
+1 -1
View File
@@ -461,7 +461,7 @@
...structuredClone(newSavedFlow),
path: $pathStore
} as Flow
triggersState.setTriggers([])
setDraftTriggers([])
loadingSave = false
dispatch('deploy', $pathStore)
} catch (err) {
frontend/src/lib/components/InstanceSetting.svelte
+91 -2
View File
@@ -17,11 +17,18 @@
import ObjectStoreConfigSettings from './ObjectStoreConfigSettings.svelte'
import { sendUserToast } from '$lib/toast'
import ConfirmButton from './ConfirmButton.svelte'
import { IndexSearchService, SettingService, TeamsService } from '$lib/gen'
import {
ConfigService,
IndexSearchService,
SettingService,
TeamsService,
type ListAvailablePythonVersionsResponse
} from '$lib/gen'
import { Button, SecondsInput, Skeleton } from './common'
import Password from './Password.svelte'
import { classNames } from '$lib/utils'
import Popover from './Popover.svelte'
import PopoverMelt from './meltComponents/Popover.svelte'
import Toggle from './Toggle.svelte'
import type { Writable } from 'svelte/store'
import { createEventDispatcher } from 'svelte'
@@ -30,6 +37,7 @@
import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte'
import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte'
import SimpleEditor from './SimpleEditor.svelte'
import LoadingIcon from './apps/svelte-select/lib/LoadingIcon.svelte'
import TeamSelector from './TeamSelector.svelte'
import ChannelSelector from './ChannelSelector.svelte'
@@ -39,7 +47,10 @@
export let loading = true
const dispatch = createEventDispatcher()
if (setting.fieldType == 'select' && $values[setting.key] == undefined) {
if (
(setting.fieldType == 'select' || setting.fieldType == 'select_python') &&
$values[setting.key] == undefined
) {
$values[setting.key] = 'default'
}
@@ -124,6 +135,24 @@
}
}
let pythonAvailableVersions: ListAvailablePythonVersionsResponse = []
let isPyFetching = false
async function fetch_available_python_versions() {
if (isPyFetching) return
isPyFetching = true
try {
pythonAvailableVersions = await ConfigService.listAvailablePythonVersions()
} catch (error) {
console.error('Error fetching python versions:', error)
} finally {
isPyFetching = false
}
}
if (setting.fieldType == 'select_python') {
fetch_available_python_versions()
}
async function fetchTeams() {
if (isFetching) return
isFetching = true
@@ -193,6 +222,66 @@
{/each}
</ToggleButtonGroup>
</div>
{:else if setting.fieldType == 'select_python'}
<div>
<!-- svelte-ignore a11y-label-has-associated-control -->
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">{setting.label}</span>
{#if setting.description}
<span class="text-secondary text-xs">
{@html setting.description}
</span>
{/if}
</label>
<ToggleButtonGroup bind:selected={$values[setting.key]} let:item={toggleButtonn}>
{#each setting.select_items ?? [] as item}
<ToggleButton
value={item.value ?? item.label}
label={item.label}
tooltip={item.tooltip}
item={toggleButtonn}
/>
{/each}
<PopoverMelt closeButton={!isPyFetching}>
<svelte:fragment slot="trigger">
{#if setting.select_items?.some((e) => e.label == $values[setting.key] || e.value == $values[setting.key])}
<Button
variant="border"
color="dark"
btnClasses="px-1.5 py-1.5 text-2xs bg-surface-secondary border-0"
nonCaptureEvent={true}>Select Custom</Button
>
{:else}
<Button
variant="border"
color="dark"
btnClasses="px-1.5 py-1.5 text-2xs border-0 shadow-md"
nonCaptureEvent={true}>Custom | {$values[setting.key]}</Button
>
{/if}
</svelte:fragment>
<svelte:fragment slot="content">
{#if isPyFetching}
<div class="p-4">
<LoadingIcon />
</div>
{:else}
<ToggleButtonGroup
bind:selected={$values[setting.key]}
let:item={toggleButtonn}
class="mr-10 h-full"
tabListClass="flex-wrap p-2"
>
{#each pythonAvailableVersions as item}
<ToggleButton value={item} label={item} tooltip={item} item={toggleButtonn} />
{/each}
</ToggleButtonGroup>
{/if}
</svelte:fragment>
</PopoverMelt>
</ToggleButtonGroup>
</div>
{:else}
<!-- svelte-ignore a11y-label-has-associated-control -->
<label class="block pb-2">
frontend/src/lib/components/ObjectStoreConfigSettings.svelte
+26 -1
View File
@@ -28,7 +28,14 @@
endpoint?: string
}
export let bucket_config: S3Config | AzureConfig | undefined = undefined
type AwsOidcConfig = {
type: 'AwsOidc'
bucket: string
region: string
roleArn: string
}
export let bucket_config: S3Config | AzureConfig | AwsOidcConfig | undefined = undefined
$: bucket_config?.type == 'S3' &&
bucket_config.allow_http == undefined &&
@@ -125,6 +132,7 @@
>
<Tab size="sm" value="S3">S3</Tab>
<Tab size="sm" value="Azure">Azure Blob</Tab>
<Tab size="sm" value="AwsOidc">AWS OIDC</Tab>
</Tabs>
<div class="flex flex-col gap-2 mt-2 p-2 border rounded-md">
{#if bucket_config.type === 'S3'}
@@ -210,6 +218,23 @@
>
<input type="text" bind:value={bucket_config.endpoint} />
</label>
{:else if bucket_config.type === 'AwsOidc'}
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Bucket</span>
<input type="text" placeholder="bucket-name" bind:value={bucket_config.bucket} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Region</span>
<input type="text" placeholder="region" bind:value={bucket_config.region} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Role ARN</span>
<input
type="text"
placeholder="arn:aws:iam::123456789012:role/test"
bind:value={bucket_config.roleArn}
/>
</label>
{:else}
<div>Unknown bucket type {bucket_config['type']}</div>
{/if}
frontend/src/lib/components/ScriptBuilder.svelte
+1 -1
View File
@@ -556,7 +556,7 @@
const { draft_triggers: _, ...newScript } = structuredClone(script)
savedScript = structuredClone(newScript) as NewScriptWithDraft
triggersState.setTriggers([])
setDraftTriggers([])
if (!disableHistoryChange) {
history.replaceState(history.state, '', `/scripts/edit/${script.path}`)
frontend/src/lib/components/ServiceLogsInner.svelte
+49 -9
View File
@@ -15,6 +15,9 @@
import AnsiUp from 'ansi_up'
import { scroll_into_view_if_needed_polyfill } from './multiselect/utils'
import SplitPanesOrColumnOnMobile from './splitPanes/SplitPanesOrColumnOnMobile.svelte'
import Select from './apps/svelte-select/lib/Select.svelte'
import { SELECT_INPUT_DEFAULT_STYLE } from '$lib/defaults'
import DarkModeObserver from './DarkModeObserver.svelte'
export let searchTerm: string
export let queryParseErrors: string[] = []
@@ -319,10 +322,13 @@
const buckets = res['buckets']
sumOtherDocCount = res['sum_other_doc_count']
countsPerHost = new Map(buckets.map(({ key, doc_count }) => [key, doc_count]))
countsPerHost = buckets.reduce((acc: any, { key, doc_count }) => {
acc[key] = { doc_count }
return acc
}, {} as Record<string, number>)
countsPerHost = buckets.reduce(
(acc: any, { key, doc_count }) => {
acc[key] = { doc_count }
return acc
},
{} as Record<string, number>
)
queryParseErrors = countLogsResponse.query_parse_errors ?? []
loadingLogCounts = false
}
@@ -376,7 +382,7 @@
let ret = {}
for (const hk of Object.keys(countsPerHost)) {
let u = hk.split(",")
let u = hk.split(',')
let [mode, wg, hn] = [u[0], u[1], u[2]]
if (!ret[mode]) {
@@ -392,8 +398,23 @@
return ret
}
function getSelectItems(allLogs: ByMode, countsPerHost: any): { label: string; value: any }[] {
return Object.entries(allLogsOrQueryResults(allLogs, countsPerHost)).flatMap(([mode, o1]) =>
Object.entries(o1).flatMap(([wg, o2]) =>
Object.keys(o2).map((hn) => ({
label: hn,
value: [mode, wg, hn]
}))
)
)
}
let darkMode = false
</script>
<DarkModeObserver bind:darkMode />
<Drawer bind:this={logDrawer} bind:open={logDrawerOpen} size="1400px">
<DrawerContent title="See context" on:close={logDrawer.closeDrawer}>
<svelte:fragment slot="actions">
@@ -436,7 +457,7 @@
month: '2-digit',
hour: '2-digit',
minute: '2-digit'
})
})
: 'min datetime'}
disabled
/>
@@ -476,7 +497,7 @@
month: '2-digit',
hour: '2-digit',
minute: '2-digit'
})
})
: 'max datetime'}
disabled
/>
@@ -548,6 +569,24 @@
>
</div>
{/if}
<div class="mr-0.5">
<Select
justValue={selected}
items={getSelectItems(allLogs, countsPerHost)}
on:change={({ detail }) => {
// console.log(detail)
selected = detail.value
}}
on:clear={() => {
selected = undefined
}}
placeholder="Select a service"
inputStyles={SELECT_INPUT_DEFAULT_STYLE.inputStyles}
containerStyles={darkMode
? SELECT_INPUT_DEFAULT_STYLE.containerStylesDark
: SELECT_INPUT_DEFAULT_STYLE.containerStyles}
/>
</div>
{#each Object.entries(allLogsOrQueryResults(allLogs, countsPerHost)) as [mode, o1]}
<div class="w-full pb-8">
<h2 class="pb-2 text-2xl">{mode}s</h2>
@@ -587,7 +626,8 @@
<div
class="text-sm pt-2 pl-0.5 whitespace-nowrap"
title={hn}
style="width: 90px;">{truncateRev(hn, countsPerHost || loadingLogs ? 40 : 8)}</div
style="width: 90px;"
>{truncateRev(hn, countsPerHost || loadingLogs ? 40 : 8)}</div
>
{#if loadingLogCounts}
<Loader2 size={15} class="animate-spin" />
@@ -752,7 +792,7 @@
month: '2-digit',
hour: '2-digit',
minute: '2-digit'
})
})
: ''}
disabled
/><CalendarPicker bind:date={upTo} label="Logs up to" /></div
frontend/src/lib/components/SimpleEditor.svelte
+31 -18
View File
@@ -467,33 +467,46 @@
}
function addTailwindClassCompletions() {
// Define a custom word definition for Tailwind classes
languages.setMonarchTokensProvider('tailwindcss', {
tokenizer: {
root: [[/[a-zA-Z0-9-]+/, 'tailwind-class']]
}
})
languages.registerCompletionItemProvider('tailwindcss', {
triggerCharacters: ['-'],
provideCompletionItems: function (model, position, context, token) {
const word = model.getWordUntilPosition(position)
const wordUntilPosition = model.getWordUntilPosition(position)
const lineContent = model.getLineContent(position.lineNumber)
// Get the text from the start of the line to the cursor
const textUntilPosition = lineContent.substring(0, position.column - 1)
// Find the last space before the cursor
const lastSpaceIndex = textUntilPosition.lastIndexOf(' ')
const startColumn = lastSpaceIndex === -1 ? 1 : lastSpaceIndex + 2
const range = {
startLineNumber: position.lineNumber,
startColumn: word.startColumn,
startColumn: startColumn,
endLineNumber: position.lineNumber,
endColumn: word.endColumn
endColumn: position.column
}
if (word && word.word) {
const currentWord = word.word
const currentWord = wordUntilPosition.word
const suggestions = tailwindClasses
.filter((className) => className.includes(currentWord))
.map((className) => ({
label: className,
kind: languages.CompletionItemKind.Class,
insertText: className,
documentation: 'Custom CSS class',
range: range
}))
const suggestions = tailwindClasses
.filter((className) => className.includes(currentWord))
.map((className) => ({
label: className,
kind: languages.CompletionItemKind.Class,
insertText: className,
documentation: 'Tailwind CSS class',
range: range,
preselect: true
}))
return { suggestions }
}
return { suggestions: [] }
return { suggestions }
}
})
}
frontend/src/lib/components/apps/components/buttons/AppButton.svelte
+4 -2
View File
@@ -229,7 +229,8 @@
css?.button?.class ?? '',
isMenuItem ? 'flex items-center justify-start' : '',
isMenuItem ? '!border-0' : '',
'wm-button'
'wm-button',
`wm-button-${resolvedConfig.color}`
)}
variant={isMenuItem ? 'border' : 'contained'}
style={css?.button?.style}
@@ -237,7 +238,8 @@
css?.container?.class ?? '',
resolvedConfig.fillContainer ? 'w-full h-full' : '',
isMenuItem ? 'w-full' : '',
'wm-button-container'
'wm-button-container',
`wm-button-container-${resolvedConfig.color}`
)}
wrapperStyle={css?.container?.style}
disabled={resolvedConfig.disabled}
frontend/src/lib/components/apps/components/display/AppAlert.svelte
+5 -4
View File
@@ -10,6 +10,7 @@
import InitializeComponent from '../helpers/InitializeComponent.svelte'
import { Alert } from '$lib/components/common'
import AlignWrapper from '../helpers/AlignWrapper.svelte'
import { appendClass } from '../../editor/componentsPanel/cssUtils'
export let id: string
export let configuration: RichConfigurations
@@ -63,13 +64,13 @@
tooltip={resolvedConfig.tooltip}
size={resolvedConfig.size}
collapsible={resolvedConfig.collapsible}
bgClass={css?.background?.class}
bgClass={appendClass(css?.background?.class, 'wm-alert-card-background')}
bgStyle={css?.background?.style}
iconClass={css?.icon?.class}
iconClass={appendClass(css?.icon?.class, 'wm-alert-card-icon')}
iconStyle={css?.icon?.style}
titleClass={css?.title?.class}
titleClass={appendClass(css?.title?.class, 'wm-alert-card-title')}
titleStyle={css?.title?.style}
descriptionClass={css?.description?.class}
descriptionClass={appendClass(css?.description?.class, 'wm-alert-card-description')}
descriptionStyle={css?.description?.style}
isCollapsed={resolvedConfig.initiallyCollapsed}
>
frontend/src/lib/components/apps/components/display/AppMenu.svelte
+1
View File
@@ -103,6 +103,7 @@
justifyEnd={false}
class={resolvedConfig.fillContainer ? 'w-full h-full' : ''}
usePointerDownOutside={true}
renderContent
>
<svelte:fragment slot="trigger" let:trigger>
<MeltButton meltElement={trigger} class="w-full h-full">
frontend/src/lib/components/apps/components/layout/AppModal.svelte
+2 -2
View File
@@ -179,7 +179,7 @@
>
<div
style={css?.popup?.style}
class={twMerge('mx-24 mt-8 bg-surface rounded-lg relative', css?.popup?.class)}
class={twMerge('mx-24 mt-8 bg-surface wm-modal rounded-lg relative', css?.popup?.class)}
use:clickOutside={{
capture: false,
stopPropagation: false,
@@ -209,7 +209,7 @@
</div>
<div
class={twMerge('wm-modal h-full', 'overflow-y-auto')}
class={twMerge('wm-modal-container h-full', 'overflow-y-auto', css?.container?.class)}
on:pointerdown={(e) => {
e?.stopPropagation()
if (!$connectingInput.opened) {
frontend/src/lib/components/apps/editor/AppEditor.svelte
-1
View File
@@ -786,7 +786,6 @@
</script>
<svelte:head>
<link rel="stylesheet" href="/tailwind_full.css" />
</svelte:head>
<DarkModeObserver on:change={onThemeChange} />
frontend/src/lib/components/apps/editor/AppPreview.svelte
-1
View File
@@ -243,7 +243,6 @@
</script>
<svelte:head>
<link rel="stylesheet" href="/tailwind_full.css" />
</svelte:head>
<DarkModeObserver on:change={onThemeChange} />
frontend/src/lib/components/apps/editor/component/components.ts
+2 -1
View File
@@ -3410,7 +3410,8 @@ See date-fns format for more information. By default, it is 'dd.MM.yyyy HH:mm'
customCss: {
button: { class: '', style: '' },
buttonContainer: { class: '', style: '' },
popup: { class: '', style: '' }
popup: { class: '', style: '' },
container: { class: '', style: '' }
},
initialData: {
horizontalAlignment: 'center',
frontend/src/lib/components/apps/editor/componentsPanel/CssHelperPanel.svelte
+140 -116
View File
@@ -20,20 +20,38 @@
const dispatch = createEventDispatcher()
interface CustomCSSEntry {
type: CustomCSSType
type?: CustomCSSType
name: string
icon: any
ids: { id: string; forceStyle: boolean; forceClass: boolean }[]
ids?: { id: string; forceStyle: boolean; forceClass: boolean }[]
description?: string
order?: number
}
const { app } = getContext<AppViewerContext>('AppViewerContext')
const descriptions = {
buttoncomponent:
'The button component also has additional color specific classes to allow customizing classes by color. wm-button-wrapper-blue, wm-button-container-blue, ...'
}
const entries: CustomCSSEntry[] = [
{
name: 'Dark Mode',
icon: LayoutDashboardIcon,
description:
'When in dark mode, the entire document has the .dark class applied to it. You can apply selective styling by using the .dark class: e.g. .dark .my-element { color: white; }',
order: 3
},
{
type: 'app',
name: 'App',
icon: LayoutDashboardIcon,
ids: ['viewer', 'grid', 'component'].map((id) => ({ id, forceStyle: true, forceClass: true }))
ids: ['viewer', 'grid', 'component'].map((id) => ({
id,
forceStyle: true,
forceClass: true
})),
order: 2
},
{
type: 'quillcomponent',
@@ -51,11 +69,12 @@
id,
forceStyle: v?.style != undefined,
forceClass: v?.['class'] != undefined
}))
})),
description: descriptions[type as keyof typeof descriptions]
}))
]
entries.sort((a, b) => a.name.localeCompare(b.name))
entries.sort((a, b) => (b.order ?? 0) - (a.order ?? 0) + a.name.localeCompare(b.name))
let search = ''
</script>
@@ -66,15 +85,15 @@
<div class="h-[calc(100%-50px)] overflow-auto relative">
{#each search != '' ? entries.filter((x) => x.name
.toLowerCase()
.includes(search.toLowerCase())) : entries as { type, name, icon, ids } (name + type)}
{#if ids.length > 0}
.includes(search.toLowerCase())) : entries as { type, name, icon, ids, description } (name + type)}
{#if description || (ids && ids.length > 0)}
<ListItem
title={name}
prefix={TITLE_PREFIX}
on:open={(e) => {
if ($app.css != undefined) {
if (e.detail && $app.css[type] == undefined) {
$app.css[type] = Object.fromEntries(ids.map(({ id }) => [id, {}]))
if (type && e.detail && $app.css[type] == undefined) {
$app.css[type] = Object.fromEntries((ids ?? []).map(({ id }) => [id, {}]))
}
}
}}
@@ -85,115 +104,120 @@
{name}
</span>
</div>
<div class="py-2">
{#each customisationByComponent.filter( (c) => c.components.includes(type) ) as customisation (customisation.components.join('-'))}
{#if customisation.link}
<a
href={customisation.link}
target="_blank"
class="text-frost-500 dark:text-frost-300 font-semibold text-xs"
>
<div class="flex flex-row gap-2">
See documentation
<ExternalLink size="16" />
</div>
</a>
{/if}
<Tabs selected="selectors">
{#if customisation.selectors.length > 0}
<Tab value="selectors" size="xs">
Selectors ({customisation.selectors.length})
</Tab>
{/if}
{#if customisation.variables.length > 0}
<Tab value="variables" size="xs">
<div class="flex flex-row gap-2 justify-center-center items-center">
Variables ({customisation.variables.length})
{#if description}
<div class="py-2 text-xs text-gray-500">{description}</div>
{/if}
{#if type}
<div class="py-2">
{#each customisationByComponent.filter( (c) => c.components.includes(type) ) as customisation (customisation.components.join('-'))}
{#if customisation.link}
<a
href={customisation.link}
target="_blank"
class="text-frost-500 dark:text-frost-300 font-semibold text-xs"
>
<div class="flex flex-row gap-2">
See documentation
<ExternalLink size="16" />
</div>
</Tab>
</a>
{/if}
<div slot="content" class="h-full">
<TabContent value="selectors" class="h-full mt-2 ">
<DataTable size="sm">
<Head>
<tr>
<Cell head first>Selector</Cell>
<Cell head>Comment</Cell>
<Cell head last />
</tr>
</Head>
{#each customisation.selectors as { selector, comment }}
<Row>
<Cell first>
<Badge color="gray">{selector}</Badge>
</Cell>
<Cell>
{#if comment}
<div class="max-w-24 whitespace-pre-wrap">{comment}</div>
{/if}
</Cell>
<Cell>
<Button
size="xs2"
color="light"
on:click={() => {
dispatch('insertSelector', `${selector} {}`)
}}
>
<ArrowUpSquare size={16} />
</Button>
</Cell>
</Row>
{/each}
</DataTable>
</TabContent>
<TabContent value="variables" class="h-full mt-2">
<DataTable>
<Head>
<tr>
<Cell head first>Variable</Cell>
<Cell head>Default value</Cell>
<Cell head>Comment</Cell>
<Cell head last />
</tr>
</Head>
{#each customisation.variables as { variable, value, comment }}
<Row>
<Cell first>
<Badge color="gray">{variable}</Badge>
</Cell>
<Cell>
<Badge color="gray">{value}</Badge>
</Cell>
<Cell>
{#if comment}
<div class="w-80 whitespace-pre-wrap">{comment}</div>
{/if}
</Cell>
<Cell sticky>
<Button
size="xs2"
color="light"
on:click={() => {
dispatch(
'insertSelector',
`${customisation.root} { ${variable}: ${value};}`
)
}}
wrapperClasses="px-2 py-3.5 bg-surface"
>
<ArrowUpSquare size={16} />
</Button>
</Cell>
</Row>
{/each}
</DataTable>
</TabContent>
</div>
</Tabs>
{/each}
</div>
<Tabs selected="selectors">
{#if customisation.selectors.length > 0}
<Tab value="selectors" size="xs">
Selectors ({customisation.selectors.length})
</Tab>
{/if}
{#if customisation.variables.length > 0}
<Tab value="variables" size="xs">
<div class="flex flex-row gap-2 justify-center-center items-center">
Variables ({customisation.variables.length})
</div>
</Tab>
{/if}
<div slot="content" class="h-full">
<TabContent value="selectors" class="h-full mt-2 ">
<DataTable size="sm">
<Head>
<tr>
<Cell head first>Selector</Cell>
<Cell head>Comment</Cell>
<Cell head last />
</tr>
</Head>
{#each customisation.selectors as { selector, comment }}
<Row>
<Cell first>
<Badge color="gray">{selector}</Badge>
</Cell>
<Cell>
{#if comment}
<div class="max-w-24 whitespace-pre-wrap">{comment}</div>
{/if}
</Cell>
<Cell>
<Button
size="xs2"
color="light"
on:click={() => {
dispatch('insertSelector', `${selector} {}`)
}}
>
<ArrowUpSquare size={16} />
</Button>
</Cell>
</Row>
{/each}
</DataTable>
</TabContent>
<TabContent value="variables" class="h-full mt-2">
<DataTable>
<Head>
<tr>
<Cell head first>Variable</Cell>
<Cell head>Default value</Cell>
<Cell head>Comment</Cell>
<Cell head last />
</tr>
</Head>
{#each customisation.variables as { variable, value, comment }}
<Row>
<Cell first>
<Badge color="gray">{variable}</Badge>
</Cell>
<Cell>
<Badge color="gray">{value}</Badge>
</Cell>
<Cell>
{#if comment}
<div class="w-80 whitespace-pre-wrap">{comment}</div>
{/if}
</Cell>
<Cell sticky>
<Button
size="xs2"
color="light"
on:click={() => {
dispatch(
'insertSelector',
`${customisation.root} { ${variable}: ${value};}`
)
}}
wrapperClasses="px-2 py-3.5 bg-surface"
>
<ArrowUpSquare size={16} />
</Button>
</Cell>
</Row>
{/each}
</DataTable>
</TabContent>
</div>
</Tabs>
{/each}
</div>
{/if}
</ListItem>
{/if}
{/each}
frontend/src/lib/components/apps/editor/componentsPanel/cssUtils.ts
+31 -1
View File
@@ -185,7 +185,11 @@ export const customisationByComponent: Customisation[] = [
components: ['modalcomponent'],
selectors: [
{ selector: '.wm-modal', comment: 'main modal element', customCssKey: 'popup' },
{ selector: '.wm-modal-button', comment: 'button to open modal', customCssKey: 'button' },
{
selector: '.wm-modal-container',
comment: 'container for modal',
customCssKey: 'container'
},
{
selector: '.wm-modal-button-container',
comment: 'container for button to open modal',
@@ -826,6 +830,26 @@ export const customisationByComponent: Customisation[] = [
selector: 'wm-alert-card-container',
comment: 'Alert container',
customCssKey: 'container'
},
{
selector: 'wm-alert-card-background',
comment: 'Alert background',
customCssKey: 'background'
},
{
selector: 'wm-alert-card-icon',
comment: 'Alert icon',
customCssKey: 'icon'
},
{
selector: 'wm-alert-card-title',
comment: 'Alert title',
customCssKey: 'title'
},
{
selector: 'wm-alert-card-description',
comment: 'Alert description',
customCssKey: 'description'
}
],
variables: []
@@ -860,3 +884,9 @@ export function hasStyleValue(obj: ComponentCssProperty | undefined) {
return obj.style !== ''
}
export function appendClass(className: string | undefined, customCssKey: string) {
if (!className) return customCssKey
return `${className} ${customCssKey}`
}
frontend/src/lib/components/apps/svelte-select/lib/Select.svelte
+8 -7
View File
@@ -1,4 +1,4 @@
<script>
<script lang="ts">
// @ts-nocheck
import { beforeUpdate, createEventDispatcher, onDestroy, onMount } from 'svelte'
import { offset, flip, shift } from '@floating-ui/dom'
@@ -22,13 +22,13 @@
export let portal = true
export let justValue = null // read-only
export let justValue: any = null // read-only
export let inAppEditor = false
let PortalWrapper = inAppEditor ? ConditionalPortal : ConditionalPortalGlobal
export let filter = _filter
export let filter: (args: any) => any[] = _filter
export let getItems = _getItems
export let id = null
@@ -38,18 +38,19 @@
export let disabled = false
export let focused = false
export let value = undefined
export let value: any = undefined
export let filterText = ''
export let placeholder = 'Please select'
export let items = undefined
export let items: { label: string | undefined; value: any }[] | any[] | string[] | undefined =
undefined
export let label = 'label'
export let itemFilter = (label, filterText, option) =>
`${label}`.toLowerCase().includes(filterText.toLowerCase())
export let groupBy = undefined
export let groupBy: ((item: any) => string) | undefined = undefined
export let groupFilter = (groups) => groups
export let groupHeaderSelectable = false
export let itemId = 'value'
export let loadOptions = undefined
export let loadOptions: ((string) => Promise<any>) | undefined = undefined
export let containerStyles = ''
export let hasError = false
export let filterSelectedItems = true
frontend/src/lib/components/instanceSettings.ts
+2 -1
View File
@@ -19,6 +19,7 @@ export interface Setting {
| 'boolean'
| 'password'
| 'select'
| 'select_python'
| 'textarea'
| 'codearea'
| 'seconds'
@@ -231,7 +232,7 @@ export const settings: Record<string, Setting[]> = {
label: 'Instance Python Version',
description: 'Default python version for newly deployed scripts',
key: 'instance_python_version',
fieldType: 'select',
fieldType: 'select_python',
// To change latest stable version:
// 1. Change placeholder in instanceSettings.ts
// 2. Change LATEST_STABLE_PY in dockerfile
frontend/src/lib/components/meltComponents/Menu.svelte
+2 -1
View File
@@ -17,6 +17,7 @@
export let usePointerDownOutside: boolean = false
export let menuClass: string = ''
export let open = false
export let renderContent: boolean = false
// Use the passed createMenu function
const menu = createMenu({
@@ -69,7 +70,7 @@
</button>
<!--svelte-ignore a11y-no-static-element-interactions-->
{#if open}
{#if open || renderContent}
<div
use:melt={$menuElement}
data-menu
frontend/src/lib/components/search/GlobalSearchModal.svelte
+72 -201
View File
@@ -3,16 +3,17 @@
import {
AppService,
FlowService,
IndexSearchService,
RawAppService,
ScriptService,
type Flow,
type ListableApp,
type ListableRawApp,
type Script
type Script,
type SearchJobsIndexResponse
} from '$lib/gen'
import { clickOutside, displayDateOnly, isMac, sendUserToast } from '$lib/utils'
import TimeAgo from '../TimeAgo.svelte'
import { clickOutside, isMac } from '$lib/utils'
import {
AlertTriangle,
BoxesIcon,
@@ -22,14 +23,12 @@
DollarSignIcon,
HomeIcon,
LayoutDashboardIcon,
Loader2,
PlayIcon,
Route,
Search,
SearchCode,
Unplug
} from 'lucide-svelte'
import JobPreview from '../runs/JobPreview.svelte'
import Portal from '$lib/components/Portal.svelte'
import { twMerge } from 'tailwind-merge'
@@ -44,6 +43,7 @@
import Popover from '../Popover.svelte'
import Logs from 'lucide-svelte/icons/logs'
import { AwsIcon, GoogleCloudIcon, KafkaIcon, MqttIcon, NatsIcon } from '../icons'
import RunsSearch from './RunsSearch.svelte'
let open: boolean = false
@@ -72,7 +72,7 @@
let switchModeItems: quickMenuItem[] = [
{
search_id: 'switchto:run-search',
label: 'Search across completed runs' + ($enterpriseLicense ? '' : ' (EE)'),
label: 'Search across completed runs' + (!$enterpriseLicense ? '' : ' (EE)'),
action: () => switchMode('runs'),
shortcutKey: RUNS_PREFIX,
icon: Search,
@@ -93,56 +93,56 @@
{
search_id: 'nav:http_routes',
label: 'Go to HTTP routes',
action: () => gotoPage('/routes'),
action: (newtab: boolean = false) => gotoPage('/routes', newtab),
icon: Route,
disabled: $userStore?.operator
},
{
search_id: 'nav:web_sockets',
label: 'Go to WebSockets',
action: () => gotoPage('/websocket_triggers'),
action: (newtab: boolean = false) => gotoPage('/websocket_triggers', newtab),
icon: Unplug,
disabled: $userStore?.operator
},
{
search_id: 'nav:postgres_triggers',
label: 'Go to Postgres triggers',
action: () => gotoPage('/postgres_triggers'),
action: (newtab: boolean = false) => gotoPage('/postgres_triggers', newtab),
icon: Database,
disabled: $userStore?.operator
},
{
search_id: 'nav:kafka_triggers',
label: 'Go to Kafka triggers' + ($enterpriseLicense ? '' : ' (EE)'),
action: () => gotoPage('/kafka_triggers'),
label: 'Go to Kafka triggers' + (!$enterpriseLicense ? '' : ' (EE)'),
action: (newtab: boolean = false) => gotoPage('/kafka_triggers', newtab),
icon: KafkaIcon,
disabled: $userStore?.operator
},
{
search_id: 'nav:nats_triggers',
label: 'Go to NATS triggers' + ($enterpriseLicense ? '' : ' (EE)'),
action: () => gotoPage('/nats_triggers'),
label: 'Go to NATS triggers' + (!$enterpriseLicense ? '' : ' (EE)'),
action: (newtab: boolean = false) => gotoPage('/nats_triggers', newtab),
icon: NatsIcon,
disabled: $userStore?.operator
},
{
search_id: 'nav:sqs_triggers',
label: 'Go to SQS triggers' + ($enterpriseLicense ? '' : ' (EE)'),
action: () => gotoPage('/sqs_triggers'),
label: 'Go to SQS triggers' + (!$enterpriseLicense ? '' : ' (EE)'),
action: (newtab: boolean = false) => gotoPage('/sqs_triggers', newtab),
icon: AwsIcon,
disabled: $userStore?.operator
},
{
search_id: 'nav:gcp_pub_sub',
label: 'Go to GCP Pub/Sub' + ($enterpriseLicense ? '' : ' (EE)'),
action: () => gotoPage('/gcp_triggers'),
label: 'Go to GCP Pub/Sub' + (!$enterpriseLicense ? '' : ' (EE)'),
action: (newtab: boolean = false) => gotoPage('/gcp_triggers', newtab),
icon: GoogleCloudIcon,
disabled: $userStore?.operator
},
{
search_id: 'nav:mqtt_triggers',
label: 'Go to MQTT triggers',
action: () => gotoPage('/mqtt_triggers'),
action: (newtab: boolean = false) => gotoPage('/mqtt_triggers', newtab),
icon: MqttIcon,
disabled: $userStore?.operator
}
@@ -152,35 +152,35 @@
{
search_id: 'nav:home',
label: 'Go to Home',
action: () => gotoPage('/'),
action: (newtab: boolean = false) => gotoPage('/', newtab),
icon: HomeIcon,
disabled: false
},
{
search_id: 'nav:runs',
label: 'Go to Runs',
action: () => gotoPage('/runs'),
action: (newtab: boolean = false) => gotoPage('/runs', newtab),
icon: PlayIcon,
disabled: false
},
{
search_id: 'nav:variables',
label: 'Go to Variables',
action: () => gotoPage('/variables'),
action: (newtab: boolean = false) => gotoPage('/variables', newtab),
icon: DollarSignIcon,
disabled: false
},
{
search_id: 'nav:resources',
label: 'Go to Resources',
action: () => gotoPage('/resources'),
action: (newtab: boolean = false) => gotoPage('/resources', newtab),
icon: BoxesIcon,
disabled: false
},
{
search_id: 'nav:schedules_triggers',
label: 'Go to Schedules',
action: () => gotoPage('/schedules'),
action: (newtab: boolean = false) => gotoPage('/schedules', newtab),
icon: CalendarIcon,
disabled: false
},
@@ -188,7 +188,7 @@
{
search_id: 'nav:service_logs',
label: 'Explore windmill service logs',
action: () => gotoPage('/service_logs'),
action: (newtab: boolean = false) => gotoPage('/service_logs', newtab),
shortcutKey: LOGS_PREFIX,
icon: Logs,
disabled: !$devopsRole
@@ -264,12 +264,8 @@
return r
}
let debounceTimeout: any = undefined
const debouncePeriod: number = 1000
let loadingCompletedRuns: boolean = false
let queryParseErrors: string[] = []
let indexMetadata: any = {}
async function handleSearch() {
queryParseErrors = []
@@ -314,6 +310,7 @@
)
)
}
itemMap['default'] = itemMap['default'].filter((e) => !e.disabled)
}
if (tab === 'switch-mode') {
itemMap['switch-mode'] = fuzzyFilter(
@@ -323,26 +320,8 @@
)
}
if (tab === 'runs') {
const s = removePrefix(searchTerm, RUNS_PREFIX)
clearTimeout(debounceTimeout)
loadingCompletedRuns = true
debounceTimeout = setTimeout(async () => {
clearTimeout(debounceTimeout)
let searchResults
try {
searchResults = await IndexSearchService.searchJobsIndex({
searchQuery: s,
workspace: $workspaceStore!
})
itemMap['runs'] = searchResults.hits
queryParseErrors = searchResults.query_parse_errors
indexMetadata = searchResults.index_metadata
} catch (e) {
sendUserToast(e.body, true)
}
loadingCompletedRuns = false
selectedItem = selectItem(0)
}, debouncePeriod)
await tick()
runsSearch?.handleRunSearch(removePrefix(searchTerm, RUNS_PREFIX))
}
selectedItem = selectItem(0)
}
@@ -407,7 +386,7 @@
textInput.focus()
}
function gotoWindmillItemPage(e: TableAny) {
function gotoWindmillItemPage(e: TableAny, newtab: boolean = false) {
let path: string
switch (e.type) {
case 'flow':
@@ -425,13 +404,17 @@
default:
path = '/'
}
gotoPage(path)
gotoPage(path, newtab)
}
function gotoPage(path: string) {
open = false
function gotoPage(path: string, newtab: boolean = false) {
searchTerm = ''
goto(path)
if (!newtab) {
open = false
goto(path)
} else {
window.open(path, "_blank")
}
}
let mouseMoved: boolean = false
@@ -594,6 +577,12 @@
return 'max-h-[60vh]'
}
}
let runsSearch: RunsSearch
let runSearchRemainingCount: number | undefined = undefined
let runSearchTotalCount: number | undefined = undefined
let indexMetadata: SearchJobsIndexResponse["index_metadata"] = undefined
</script>
{#if open}
@@ -652,18 +641,16 @@
{#if items.length > 0}
<div class={tab === 'switch-mode' ? 'p-2' : 'p-2 border-b'}>
{#each items as el}
{#if !el.disabled}
<QuickMenuItem
on:select={el?.action}
on:hover={() => (selectedItem = el)}
id={el?.search_id}
hovered={el?.search_id === selectedItem?.search_id}
label={el?.label}
icon={el?.icon}
shortcutKey={el?.shortcutKey}
bind:mouseMoved
/>
{/if}
<QuickMenuItem
onselect={(shift) => el?.action(shift)}
onhover={() => (selectedItem = el)}
id={el?.search_id}
hovered={el?.search_id === selectedItem?.search_id}
label={el?.label}
icon={el?.icon}
shortcutKey={el?.shortcutKey}
bind:mouseMoved
/>
{/each}
</div>
{/if}
@@ -677,8 +664,10 @@
</div>
{#each (itemMap[tab] ?? []).filter((e) => (combinedItems ?? []).includes(e)) as el}
<QuickMenuItem
on:select={() => gotoWindmillItemPage(el)}
on:hover={() => (selectedItem = el)}
onselect={(shift) => {
gotoWindmillItemPage(el, shift)
}}
onhover={() => (selectedItem = el)}
id={el?.search_id}
hovered={el?.path === selectedItem?.path}
label={(el.summary ? `${el.summary} - ` : '') +
@@ -715,7 +704,7 @@
</Alert>
{:else}
<QuickMenuItem
on:select={() =>
onselect={() =>
gotoPage(
`/service_logs?query=${encodeURIComponent(removePrefix(searchTerm, '!'))}`
)}
@@ -729,138 +718,20 @@
{/if}
</div>
{:else if tab === 'runs'}
<div class="flex h-full p-2 divide-x">
{#if loadingCompletedRuns}
<div class="flex w-full justify-center items-center h-48">
<div class="text-tertiary text-center">
<Loader2 size={34} class="animate-spin" />
</div>
</div>
{:else if itemMap['runs'] && itemMap['runs'].length > 0}
<div class="w-4/12 overflow-y-auto max-h-[70vh]">
{#each itemMap['runs'] ?? [] as r}
<QuickMenuItem
on:select={() => {
selectedItem = r
selectedWorkspace = r?.document.workspace_id[0]
}}
on:keyboardOnlySelect={() => {
open = false
goto(`/run/${r?.document.id[0]}`)
}}
id={r?.document.id[0]}
hovered={selectedItem && r?.document.id[0] === selectedItem?.document.id[0]}
icon={r?.icon}
containerClass="rounded-md px-2 py-1 my-2"
bind:mouseMoved
>
<svelte:fragment slot="itemReplacement">
<div
class="w-full flex flex-row items-center gap-4 transition-all"
>
<div
class="rounded-full w-2 h-2 {r?.document.success[0]
? 'bg-green-400'
: 'bg-red-400'}"
></div>
<div class="flex flex-col gap-2">
<div class="text-xs"> {r?.document.script_path} </div>
<div class="flex flex-row gap-2">
<div
class="whitespace-nowrap col-span-2 !text-tertiary !text-2xs overflow-hidden text-ellipsis flex-shrink text-center"
>
{displayDateOnly(new Date(r?.document.created_at[0]))}
</div>
<div
class="whitespace-nowrap col-span-2 !text-tertiary !text-2xs overflow-hidden text-ellipsis flex-shrink text-center"
>
<TimeAgo date={r?.document.created_at[0] ?? ''} />
</div>
</div>
</div>
</div>
</svelte:fragment>
</QuickMenuItem>
{/each}
</div>
<div class="w-8/12 max-h-[70vh]">
{#if selectedItem === undefined}
Select a result to preview
{:else}
<div class="h-[95%] overflow-y-scroll">
<JobPreview
id={selectedItem?.document?.id[0]}
workspace={selectedWorkspace}
/>
</div>
{/if}
<div class="flex flex-row pt-3 pl-4 items-center text-xs text-secondary">
{#if indexMetadata.indexed_until}
<span class="px-2">
Most recently indexed job was created at <TimeAgo
agoOnlyIfRecent
date={indexMetadata.indexed_until || ''}
/>
</span>
{/if}
{#if indexMetadata.lost_lock_ownership}
<Popover notClickable placement="top">
<AlertTriangle size={16} class="text-gray-500" />
<svelte:fragment slot="text">
The current indexer is no longer indexing new jobs. This is most likely
because of an ongoing deployment and indexing will resume once it's
complete.
</svelte:fragment>
</Popover>
{/if}
</div>
</div>
{:else}
<div class="flex flex-col h-full w-full justify-center items-center h-48">
<div class="text-tertiary text-center">
{#if searchTerm === RUNS_PREFIX}
<div class="text-2xl font-bold">Enter your search terms</div>
<div class="text-sm"
>Start typing to do full-text search across completed runs</div
>
{:else}
<div class="text-2xl font-bold">No runs found</div>
<div class="text-sm">There were no completed runs that match your query</div>
{/if}
<div class="text-sm">
Note that new runs might take a while to become searchable (by default ~5min)
</div>
{#if !$enterpriseLicense}
<div class="py-6"></div>
<Alert title="This is an EE feature" type="warning">
Full-text search on jobs is only available on EE.
</Alert>
{/if}
</div>
<div class="flex flex-row pt-10 text-xs text-secondary">
{#if indexMetadata.indexed_until}
<span class="px-2">
Most recently indexed job was created at <TimeAgo
agoOnlyIfRecent
date={indexMetadata.indexed_until}
/>
</span>
{/if}
{#if indexMetadata.lost_lock_ownership}
<Popover notClickable placement="top">
<AlertTriangle size={16} class="text-gray-500" />
<svelte:fragment slot="text">
The current indexer is no longer indexing new jobs. This is most likely
because of an ongoing deployment and indexing will resume once it's
complete.
</svelte:fragment>
</Popover>
{/if}
</div>
</div>
{/if}
</div>
<RunsSearch
bind:queryParseErrors
bind:this={runsSearch}
bind:selectedItem
bind:selectedWorkspace
bind:mouseMoved
bind:loadedRuns={itemMap['runs']}
bind:open
{selectItem}
searchTerm={removePrefix(searchTerm, RUNS_PREFIX)}
bind:runSearchRemainingCount
bind:runSearchTotalCount
bind:indexMetadata
/>
{/if}
</div>
</div>
frontend/src/lib/components/search/QuickMenuItem.svelte
+53 -28
View File
@@ -1,17 +1,9 @@
<script lang="ts">
import { isMac } from '$lib/utils'
import { createEventDispatcher, onDestroy, onMount } from 'svelte'
import { onDestroy, onMount } from 'svelte'
import { twMerge } from 'tailwind-merge'
export let hovered: boolean = false
export let id: string
export let label: string = ''
export let icon: any = undefined
export let shortcutKey: string | undefined = undefined
export let containerClass: string | undefined = undefined
export let mouseMoved = false
const dispatch = createEventDispatcher()
// const dispatch = createEventDispatcher()
onMount(() => {
window.addEventListener('keydown', handleKeydown)
@@ -24,16 +16,46 @@
async function handleKeydown(event: KeyboardEvent) {
if (hovered && event.key === 'Enter') {
event.preventDefault()
dispatch('keyboardOnlySelect')
runAction()
if (onkeyboardSpecificSelect) {
onkeyboardSpecificSelect(event.shiftKey || event.ctrlKey)
} else {
onselect(event.shiftKey || event.ctrlKey)
}
}
}
function runAction() {
dispatch('select')
interface Props {
hovered?: boolean
id: string
label?: string
icon?: any
shortcutKey?: string | undefined
containerClass?: string | undefined
mouseMoved?: boolean
kbdClass?: string
small?: boolean
itemReplacement?: import('svelte').Snippet
onselect?: (shift: boolean) => void
onkeyboardSpecificSelect?: (shift: boolean) => void
onhover?: () => void
}
export let kbdClass = ''
export let small = true
let {
hovered = false,
id,
label = '',
icon = undefined,
shortcutKey = undefined,
containerClass = undefined,
mouseMoved = $bindable(false),
kbdClass = $bindable(''),
small = true,
itemReplacement,
onselect = () => {},
onhover = () => {},
onkeyboardSpecificSelect
}: Props = $props()
if (small) {
kbdClass = twMerge(
kbdClass,
@@ -44,33 +66,36 @@
} else {
kbdClass += ' !text-xs px-1.5'
}
</script>
<!-- svelte-ignore a11y-no-static-element-interactions -->
<!-- svelte-ignore a11y-click-events-have-key-events -->
<!-- svelte-ignore a11y_no_static_element_interactions -->
<!-- svelte-ignore a11y_click_events_have_key_events -->
<div
{id}
on:click|stopPropagation={runAction}
on:mouseenter={() => {
onclick={(e) => {
e.stopImmediatePropagation()
onselect(e.shiftKey || e.ctrlKey)
}}
onmouseenter={() => {
if (mouseMoved) {
dispatch('hover')
onhover()
}
mouseMoved=false
mouseMoved = false
}}
class={twMerge(
`rounded-md w-full transition-all cursor-pointer ${
hovered ? 'bg-surface-hover' : ''
}`,
`rounded-md w-full transition-all cursor-pointer ${hovered ? 'bg-surface-hover' : ''}`,
containerClass
)}
>
{#if $$slots.itemReplacement}
<slot name="itemReplacement" />
{#if itemReplacement}
{@render itemReplacement?.()}
{:else}
<div class="flex flex-row gap-2 items-center px-2 py-1.5 rounded-md pr-6 text-sm">
<div class="w-4">
{#if icon}
<svelte:component this={icon} size={16} />
{@const SvelteComponent = icon}
<SvelteComponent size={16} />
{:else if shortcutKey != undefined}
<div class="font-bold flex items-center justify-center w-full">
<span
frontend/src/lib/components/search/RunsSearch.svelte
+278
View File
@@ -0,0 +1,278 @@
<script lang="ts">
import { IndexSearchService, type SearchJobsIndexResponse } from '$lib/gen'
import { enterpriseLicense, workspaceStore } from '$lib/stores'
import { sendUserToast } from '$lib/toast'
import { AlertTriangle, Loader2 } from 'lucide-svelte'
import TimeAgo from '../TimeAgo.svelte'
import Popover from '../Popover.svelte'
import { Alert } from '../common'
import QuickMenuItem from './QuickMenuItem.svelte'
import { goto } from '$app/navigation'
import { displayDateOnly } from '$lib/utils'
import JobPreview from '../runs/JobPreview.svelte'
let debounceTimeout: any = undefined
const debouncePeriod: number = 1000
let loadingCompletedRuns: boolean = $state(false)
let loadingMoreJobs: boolean = $state(false)
interface Props {
mouseMoved: boolean
selectedWorkspace: string | undefined
selectedItem: any
queryParseErrors: string[]
open: boolean
loadedRuns: any[]
selectItem: (idx: number) => any
searchTerm: string
runSearchRemainingCount: number | undefined
runSearchTotalCount: number | undefined
indexMetadata: SearchJobsIndexResponse['index_metadata']
}
let {
mouseMoved = $bindable(),
selectedWorkspace = $bindable(),
selectedItem = $bindable(),
queryParseErrors = $bindable(),
open = $bindable(),
loadedRuns = $bindable(),
selectItem,
searchTerm,
runSearchRemainingCount = $bindable(),
runSearchTotalCount = $bindable(),
indexMetadata = $bindable()
}: Props = $props()
export function handleRunSearch(s: string) {
clearTimeout(debounceTimeout)
loadingCompletedRuns = true
debounceTimeout = setTimeout(async () => {
clearTimeout(debounceTimeout)
let searchResults: SearchJobsIndexResponse
try {
searchResults = await IndexSearchService.searchJobsIndex({
searchQuery: s,
workspace: $workspaceStore!
})
if (s !== searchTerm) {
loadingCompletedRuns = false
return
}
loadedRuns = searchResults.hits ?? []
runSearchTotalCount = searchResults.hit_count
runSearchRemainingCount = (searchResults.hit_count ?? 0) - loadedRuns?.length
queryParseErrors = searchResults.query_parse_errors ?? []
indexMetadata = searchResults.index_metadata
if (runSearchRemainingCount > 0) {
loadedRuns.push({ search_id: 'opt:load_more_jobs' })
}
} catch (e) {
sendUserToast(e.body, true)
}
loadingCompletedRuns = false
selectedItem = selectItem(0)
}, debouncePeriod)
}
async function loadMoreJobs(s: string, paginationOffset: number) {
loadingMoreJobs = true
let searchResults: SearchJobsIndexResponse
try {
searchResults = await IndexSearchService.searchJobsIndex({
searchQuery: s,
paginationOffset,
workspace: $workspaceStore!
})
if (s !== searchTerm) {
loadingMoreJobs = false
return
}
loadedRuns.pop()
loadedRuns = loadedRuns.concat(searchResults.hits)
runSearchTotalCount = searchResults.hit_count
runSearchRemainingCount = (searchResults.hit_count ?? 0) - loadedRuns?.length
queryParseErrors = searchResults.query_parse_errors ?? []
indexMetadata = searchResults.index_metadata
if (runSearchRemainingCount > 0) {
loadedRuns.push({ search_id: 'opt:load_more_jobs' })
}
} catch (e) {
sendUserToast(e.body, true)
}
loadingMoreJobs = false
selectedItem = selectItem(paginationOffset)
}
</script>
<div class="flex h-full p-2 divide-x">
{#if loadingCompletedRuns}
<div class="flex w-full justify-center items-center h-48">
<div class="text-tertiary text-center">
<Loader2 size={34} class="animate-spin" />
</div>
</div>
{:else if loadedRuns && loadedRuns.length > 0}
<div class="w-4/12 max-h-[70vh] flex flex-col">
<div class="text-tertiary text-xs">
{runSearchTotalCount} jobs matched the query
</div>
<div class="overflow-y-auto">
{#each loadedRuns ?? [] as r}
{#if r.search_id === 'opt:load_more_jobs'}
<div class="pt-4"></div>
{#if loadingMoreJobs}
<div class="pl-8 pb-8 text-tertiary text-center">
<Loader2 size={20} class="animate-spin" />
</div>
{:else}
<QuickMenuItem
onselect={() => {
selectedItem = r
selectedWorkspace = undefined
const paginationOffset = runSearchTotalCount! - runSearchRemainingCount!
loadMoreJobs(searchTerm, paginationOffset)
}}
id={'opt:load_more_jobs'}
hovered={selectedItem && r?.search_id === selectedItem?.search_id}
containerClass="rounded-md px-2 py-1 my-2"
bind:mouseMoved
>
{#snippet itemReplacement()}
<div
class="py-2 w-full flex flex-row items-center gap-4 transition-all text-secondary text-sm"
>
Some other {runSearchRemainingCount} jobs matched the query. Click to load more.
<!-- Load more ({runSearchRemainingCount} other) -->
<!-- {runSearchRemainingCount} more documents also matched -->
</div>
{/snippet}
</QuickMenuItem>
{/if}
{:else}
<QuickMenuItem
onselect={(shift) => {
selectedItem = r
selectedWorkspace = r?.document.workspace_id[0]
if (shift) {
window.open(`/run/${r?.document.id[0]}`, '_blank')
}
}}
onkeyboardSpecificSelect={(shift) => {
if (!shift) {
open = false
goto(`/run/${r?.document.id[0]}`)
} else {
window.open(`/run/${r?.document.id[0]}`, '_blank')
}
}}
id={r?.document.id[0]}
hovered={selectedItem && r?.search_id === selectedItem?.search_id}
icon={r?.icon}
containerClass="rounded-md px-2 py-1 my-2"
bind:mouseMoved
>
{#snippet itemReplacement()}
<div class="w-full flex flex-row items-center gap-4 transition-all">
<div
class="rounded-full w-2 h-2 {r?.document.success[0]
? 'bg-green-400'
: 'bg-red-400'}"
></div>
<div class="flex flex-col gap-2">
<div class="text-xs"> {r?.document.script_path} </div>
<div class="flex flex-row gap-2">
<div
class="whitespace-nowrap col-span-2 !text-tertiary !text-2xs overflow-hidden text-ellipsis flex-shrink text-center"
>
{displayDateOnly(new Date(r?.document.created_at[0]))}
</div>
<div
class="whitespace-nowrap col-span-2 !text-tertiary !text-2xs overflow-hidden text-ellipsis flex-shrink text-center"
>
<TimeAgo date={r?.document.created_at[0] ?? ''} />
</div>
</div>
</div>
</div>
{/snippet}
</QuickMenuItem>
{/if}
{/each}
</div>
</div>
<div class="w-8/12 max-h-[70vh]">
{#if selectedItem === undefined}
Select a result to preview
{:else}
<div class="h-[95%] overflow-y-scroll">
<JobPreview id={selectedItem?.document?.id[0]} workspace={selectedWorkspace} />
</div>
{/if}
<div class="flex flex-row pt-3 pl-4 items-center text-xs text-secondary">
{#if indexMetadata?.indexed_until}
<span class="px-2">
Most recently indexed job was created at <TimeAgo
agoOnlyIfRecent
date={indexMetadata.indexed_until || ''}
/>
</span>
{/if}
{#if indexMetadata?.lost_lock_ownership}
<Popover notClickable placement="top">
<AlertTriangle size={16} class="text-gray-500" />
<svelte:fragment slot="text">
The current indexer is no longer indexing new jobs. This is most likely because of an
ongoing deployment and indexing will resume once it's complete.
</svelte:fragment>
</Popover>
{/if}
</div>
</div>
{:else}
<div class="flex flex-col h-full w-full justify-center items-center h-48">
<div class="text-tertiary text-center">
{#if searchTerm === ''}
<div class="text-2xl font-bold">Enter your search terms</div>
<div class="text-sm">Start typing to do full-text search across completed runs</div>
{:else}
<div class="text-2xl font-bold">No runs found</div>
<div class="text-sm">There were no completed runs that match your query</div>
{/if}
<div class="text-sm">
Note that new runs might take a while to become searchable (by default ~5min)
</div>
{#if !$enterpriseLicense}
<div class="py-6"></div>
<Alert title="This is an EE feature" type="warning">
Full-text search on jobs is only available on EE.
</Alert>
{/if}
</div>
<div class="flex flex-row pt-10 text-xs text-secondary">
{#if indexMetadata?.indexed_until}
<span class="px-2">
Most recently indexed job was created at <TimeAgo
agoOnlyIfRecent
date={indexMetadata.indexed_until}
/>
</span>
{/if}
{#if indexMetadata?.lost_lock_ownership}
<Popover notClickable placement="top">
<AlertTriangle size={16} class="text-gray-500" />
<svelte:fragment slot="text">
The current indexer is no longer indexing new jobs. This is most likely because of an
ongoing deployment and indexing will resume once it's complete.
</svelte:fragment>
</Popover>
{/if}
</div>
</div>
{/if}
</div>
frontend/tailwind.config.cjs
+3 -1
View File
@@ -1,4 +1,5 @@
const plugin = require('tailwindcss/plugin')
const { tailwindClasses } = require('./src/lib/components/apps/editor/componentsPanel/tailwindUtils')
const lightTheme = {
surface: '#ffffff',
@@ -80,7 +81,8 @@ const config = {
'autocomplete-list-item',
'autocomplete-list-item-create',
'selected',
'wm-tab-selected'
'wm-tab-selected',
...tailwindClasses
],
theme: {
colors: {
lsp/Pipfile
+2 -2
View File
@@ -4,8 +4,8 @@ verify_ssl = true
name = "pypi"
[packages]
wmill = ">=1.491.5"
wmill_pg = ">=1.491.5"
wmill = ">=1.492.1"
wmill_pg = ">=1.492.1"
sendgrid = "*"
mysql-connector-python = "*"
pymongo = "*"
openflow.openapi.yaml
+1 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.491.5
version: 1.492.1
title: OpenFlow Spec
contact:
name: Ruben Fiszel
powershell-client/WindmillClient/WindmillClient.psd1
+1 -1
View File
@@ -12,7 +12,7 @@
RootModule = 'WindmillClient.psm1'
# Version number of this module.
ModuleVersion = '1.491.5'
ModuleVersion = '1.492.1'
# Supported PSEditions
# CompatiblePSEditions = @()
python-client/wmill/pyproject.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "wmill"
version = "1.491.5"
version = "1.492.1"
description = "A client library for accessing Windmill server wrapping the Windmill client API"
license = "Apache-2.0"
homepage = "https://windmill.dev"

Some files were not shown because too many files have changed in this diff Show More