diff --git a/.cursor/rules/svelte5-best-practices.mdc b/.cursor/rules/svelte5-best-practices.mdc index c2fa3baae8..6951584bc5 100644 --- a/.cursor/rules/svelte5-best-practices.mdc +++ b/.cursor/rules/svelte5-best-practices.mdc @@ -5,7 +5,7 @@ alwaysApply: false --- # Svelte 5 Best Practices -This guide outlines best practices for developing with Svelte 5, incorporating the new Runes API and other modern Svelte features. They should be applied on every new files created, but not on existing svelte 4 files unless specifically asked to. +This guide outlines best practices for developing with Svelte 5, incorporating the new Runes API and other modern Svelte features. These rules MUST NOT be applied on svelte 4 files unless explicitly asked to do so. ## Reactivity with Runes diff --git a/.cursor/rules/windmill-overview.mdc b/.cursor/rules/windmill-overview.mdc new file mode 100644 index 0000000000..8c87dbebdc --- /dev/null +++ b/.cursor/rules/windmill-overview.mdc @@ -0,0 +1,76 @@ +--- +description: +globs: +alwaysApply: false +--- +# Windmill Overview + +Windmill is an open-source developer platform for building internal tools, API integrations, background jobs, workflows, and user interfaces. It offers a unified system where scripts are automatically turned into sharable UIs and can be composed into flows or embedded in custom applications. + +## Core Capabilities + +- **Script Development and Execution**: Write and run scripts in Python, TypeScript/JavaScript (Deno/Bun), Go, Bash, SQL, and other languages +- **Workflow Orchestration**: Compose scripts into multi-step flows with conditional logic, loops, and error handling +- **UI Generation**: Automatically generate UIs from scripts or build custom applications with a low-code editor +- **Job Scheduling**: Trigger scripts and flows on schedules, webhooks, or external events +- **Resource Management**: Securely store and use credentials, databases, and other connections + +## Platform Architecture + +The Windmill platform consists of several key components: + +- **Frontend UI**: Web-based interface for script and flow development, app building, and result visualization +- **API Server**: Central API that handles authentication, resource management, and job coordination +- **Workers**: Execute scripts in their respective environments with proper sandboxing +- **Database**: PostgreSQL database for storage of scripts, flows, resources, job results, and more +- **Job Queue**: Queue system for managing job execution, implemented in PostgreSQL +- **Client Libraries**: Libraries for interacting with Windmill from Python, TypeScript, or command line + +# Windmill Backend Architecture + +The Windmill backend is written in Rust and consists of several services working together. These services are designed for horizontal scaling with stateless API servers and workers that can be deployed across multiple machines. + +## Key Components + +- **API Server (`windmill-api`)**: Handles HTTP requests, authentication, and resource management +- **Queue Manager (`windmill-queue`)**: Manages the job queue in PostgreSQL +- **Worker System (`windmill-worker`)**: Executes jobs in sandboxed environments +- **Common Utilities (`windmill-common`)**: Shared code used by multiple services +- **Git Sync (`windmill-git-sync`)**: Synchronizes scripts with Git repositories + +## Job Execution System + +The job execution process follows these steps: + +1. The API server receives a request to run a script or flow and creates a job record in the database +2. The job is added to the queue system in PostgreSQL +3. Workers continuously poll the queue for jobs matching their capabilities +4. When a job is picked up, it's routed to the appropriate language executor +5. The script is executed in a sandboxed environment using NSJAIL for security +6. Results are processed and stored in the database +7. For flows, each step creates a new job that goes through the same process + +Windmill supports worker tags and groups to route jobs to workers with specific capabilities or resource access. + +# Windmill Frontend Architecture + +The Windmill frontend is built with Svelte and provides several key interfaces for interacting with the platform. + +## Key Components + +- **Script Builder**: Code editor with language support, schema inference, and dependency management +- **Flow Builder**: Visual editor for creating multi-step workflows with branching and looping +- **App Editor**: Grid-based editor for building custom UIs that integrate scripts and flows +- **Schema Form System**: Generates form interfaces from script parameters automatically +- **Result Viewer**: Visualizes job results, logs, and execution status + +The frontend uses the Monaco editor (same as VS Code) for code editing, with specialized language support for all supported script languages. + +## UI Framework + +The frontend is built with Svelte, providing a reactive and component-based architecture. Key frontend technologies include: + +- **Svelte/SvelteKit**: Core framework for UI components and routing +- **Monaco Editor**: Code editing experience similar to VS Code +- **Schema Form**: Automatic UI generation from TypeScript/JSON schemas +- **Tailwind CSS**: Utility-first CSS framework for styling diff --git a/.github/workflows/aider-after-review.yaml b/.github/workflows/aider-after-review.yaml index 6ae15bfc18..8347b1f9f2 100644 --- a/.github/workflows/aider-after-review.yaml +++ b/.github/workflows/aider-after-review.yaml @@ -70,6 +70,8 @@ jobs: # Get PR review body REVIEW_BODY="${{ github.event.review.body }}" + REVIEW_BODY_Q=$(printf '%q' "$REVIEW_BODY") + PR_NUMBER="${{ github.event.pull_request.number }}" # Get PR description for context NOT USED FOR NOW @@ -78,10 +80,6 @@ jobs: # PR_BODY=$(echo "$PR_DETAILS" | jq -r .body) # Get all PR review comments - REVIEW_COMMENTS=$(gh pr view $PR_NUMBER --json reviews -q '.reviews[] | select(.state == "CHANGES_REQUESTED") | .body' --repo $GITHUB_REPOSITORY) - REVIEW_BODY_Q=$(printf '%q' "$REVIEW_BODY") - - # Update query to get review comments from all review types, not just "CHANGES_REQUESTED" ALL_REVIEW_COMMENTS=$(gh api \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ @@ -89,15 +87,63 @@ jobs: | jq '[.[] | {diff_hunk: .diff_hunk, path: .path, body: .body}]') BASE_PROMPT="Fix the following issues in the PR based on the review feedback. The review body is prepended with REVIEW. The review comments are prepended with REVIEW_COMMENTS. The review body and comments are separated by a blank line." - printf "%s\nREVIEW:\n%s\nREVIEW_COMMENTS:\n%s" \ - "$BASE_PROMPT" "$REVIEW_BODY_Q" "$ALL_REVIEW_COMMENTS" > "$PROMPT_FILE_PATH" + COMPLETE_PROMPT=$(printf "%s\nREVIEW:\n%s\nREVIEW_COMMENTS:\n%s" \ + "$BASE_PROMPT" "$REVIEW_BODY_Q" "$ALL_REVIEW_COMMENTS") + echo "$COMPLETE_PROMPT" > "$PROMPT_FILE_PATH" echo "PROMPT_FILE_PATH=$PROMPT_FILE_PATH" >> $GITHUB_OUTPUT + - name: Probe Chat for Relevant Files + id: probe_files + env: + PROMPT_CONTENT_FILE: ${{ steps.generate_prompt.outputs.PROMPT_FILE_PATH }} + run: | + echo "Running probe-chat to find relevant files..." + if [[ ! -f "$PROMPT_CONTENT_FILE" ]]; then + echo "::error::Prompt file $PROMPT_CONTENT_FILE not found!" + exit 1 + fi + PROMPT_CONTENT=$(cat "$PROMPT_CONTENT_FILE") + if [ -z "$PROMPT_CONTENT" ]; then + echo "::error::Prompt content is empty!" + exit 1 + fi + + PROMPT_ESCAPED=$(jq -Rs . <<< "$PROMPT_CONTENT") + + MESSAGE_FOR_PROBE=$(jq -n --arg prompt_escaped "$PROMPT_ESCAPED" \ + '{ "message": "I'\''m giving you a request that needs to be implemented. Your role is ONLY to give me the files that are relevant to the request and nothing else. The request is prepended with the word REQUEST.\\nREQUEST: \($prompt_escaped). Give me all the files relevant to this request. Your output MUST be a single json array that can be parsed with programatic json parsing, with the relevant files. Files can be rust or typescript or javascript files. DO NOT INCLUDE ANY OTHER TEXT IN YOUR OUTPUT. ONLY THE JSON ARRAY. Example of output: [\"file1.py\", \"file2.py\"]" }' | jq -r .message) + + set -o pipefail + PROBE_OUTPUT=$(npx --yes @buger/probe-chat@latest --max-iterations 50 --model-name gemini-2.5-pro-preview-05-06 --message "$MESSAGE_FOR_PROBE") || { + echo "::error::probe-chat command failed. Output:" + echo "$PROBE_OUTPUT" + exit 1 + } + set +o pipefail + echo "Probe-chat raw output:" + echo "$PROBE_OUTPUT" + + JSON_FILES=$(echo "$PROBE_OUTPUT" | sed -n '/^\s*\[/,$p' | sed '/^\s*\]/q') + echo "Extracted JSON block:" + echo "$JSON_FILES" + + FILES_LIST=$(echo "$JSON_FILES" | jq -e -r '[.[] | select(type == "string" and . != "" and . != null and (endswith("/") | not))] | map(@sh) | join(" ")' || echo "") + + if [[ -z "$FILES_LIST" ]]; then + echo "::warning::probe-chat did not identify any relevant files." + exit 1 + fi + + echo "Formatted files list for aider: $FILES_LIST" + echo "FILES_TO_EDIT=$FILES_LIST" >> $GITHUB_ENV + - name: Run Aider with review prompt run: | aider \ --read .cursor/rules/rust-best-practices.mdc \ --read .cursor/rules/svelte5-best-practices.mdc \ + --read .cursor/rules/windmill-overview.mdc \ + ${{ env.FILES_TO_EDIT }} \ --model gemini/gemini-2.5-pro-preview-05-06 \ --message-file .github/aider/review-prompt.txt \ --yes \ @@ -110,6 +156,7 @@ jobs: # Check if there are any changes to commit if [[ -z "$(git status --porcelain)" ]]; then echo "No changes detected after running Aider." + echo "HAS_CHANGES=false" >> $GITHUB_OUTPUT exit 0 fi @@ -125,6 +172,7 @@ jobs: echo "Attempting to push changes to PR branch $CURRENT_BRANCH_NAME for PR #${{ github.event.pull_request.number }}" # Pull latest changes to avoid rejection due to non-fast-forward + git config pull.rebase true git pull origin $CURRENT_BRANCH_NAME if git push origin $CURRENT_BRANCH_NAME; then diff --git a/.github/workflows/aider.yaml b/.github/workflows/aider.yaml index 52d382eb78..82d5f8b63e 100644 --- a/.github/workflows/aider.yaml +++ b/.github/workflows/aider.yaml @@ -266,6 +266,7 @@ jobs: aider \ --read .cursor/rules/rust-best-practices.mdc \ --read .cursor/rules/svelte5-best-practices.mdc \ + --read .cursor/rules/windmill-overview.mdc \ ${{ env.FILES_TO_EDIT }} \ --model gemini/gemini-2.5-pro-preview-05-06 \ --message-file .github/aider/issue-prompt.txt \ diff --git a/.github/workflows/discord-notification.yml b/.github/workflows/discord-notification.yml index d5a23c1929..4d062f0816 100644 --- a/.github/workflows/discord-notification.yml +++ b/.github/workflows/discord-notification.yml @@ -9,7 +9,7 @@ on: jobs: notify_discord_when_pr_opened: - if: github.event.pull_request.draft == false + if: (github.event.pull_request.draft == false) && (github.event.action == 'opened' || github.event.action == 'ready_for_review') uses: ./.github/workflows/shareable-discord-notification.yml with: PR_TITLE: ${{ github.event.pull_request.title }} diff --git a/.github/workflows/linear-issue.yaml b/.github/workflows/linear-issue.yaml new file mode 100644 index 0000000000..e3fbeb7f39 --- /dev/null +++ b/.github/workflows/linear-issue.yaml @@ -0,0 +1,223 @@ +name: External Aider Issue Fix + +on: + repository_dispatch: + types: [external_issue_fix] + +jobs: + auto-fix: + runs-on: ubicloud-standard-8 + permissions: + contents: write + pull-requests: write + issues: write + env: + GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WINDMILL_TOKEN: ${{ secrets.WINDMILL_TOKEN }} + + steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + + - name: Check out code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Configure Git User + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Aider and Dependencies + run: | + python -m pip install aider-install; aider-install + pip install -U google-generativeai + sudo apt-get update && sudo apt-get install -y jq + + - name: Create Prompt for Aider + id: create_prompt + shell: bash + run: | + PROMPT_FILE_PATH=".github/aider/issue-prompt.txt" + mkdir -p .github/aider + + ISSUE_TITLE="${{ github.event.client_payload.issue_title }}" + INSTRUCTION="${{ github.event.client_payload.instruction }}" + ISSUE_BODY=$(printf '%q' "${{ github.event.client_payload.issue_body }}") + + echo "Processing issue with title: $ISSUE_TITLE" + + JSON_PAYLOAD=$(jq -n \ + --arg title "$ISSUE_TITLE" \ + --arg body "$ISSUE_BODY" \ + '{"body":{"issue_title":$title,"issue_body":$body}}') + + API_RESULT=$(curl -s -w "\n%{http_code}" \ + -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run_wait_result/p/f/ai/quiet_script" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $WINDMILL_TOKEN" \ + --data-binary "$JSON_PAYLOAD" \ + --max-time 90) + + HTTP_CODE=$(echo "$API_RESULT" | tail -n1) + BODY=$(echo "$API_RESULT" | sed '$d') + + echo "$BODY" > /tmp/api_response.txt + + BASE_PROMPT="Try to fix the following issue based on the instruction given. The issue is prepended with the word ISSUE. The instruction is prepended with the word INSTRUCTION. The issue and instruction are separated by a blank line." + if [[ "$HTTP_CODE" -eq 200 ]]; then + PROCESSED_ISSUE_PROMPT=$(jq -r '.effective_body // empty' /tmp/api_response.txt) + if [[ -z "$PROCESSED_ISSUE_PROMPT" || "$PROCESSED_ISSUE_PROMPT" == "null" ]]; then + PROCESSED_ISSUE_PROMPT="" + fi + printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \ + "$BASE_PROMPT" "$PROCESSED_ISSUE_PROMPT" "$INSTRUCTION" > "$PROMPT_FILE_PATH" + else + echo "::warning::API call failed (HTTP $HTTP_CODE). Using raw issue content." + printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \ + "$BASE_PROMPT" "$ISSUE_BODY" "$INSTRUCTION" > "$PROMPT_FILE_PATH" + fi + rm -f /tmp/api_response.txt + + echo "Prompt created and written to $PROMPT_FILE_PATH" + echo "PROMPT_FILE_PATH=$PROMPT_FILE_PATH" >> $GITHUB_OUTPUT + + # Store the issue title for PR creation + ISSUE_TITLE_SAFE=$(echo "$ISSUE_TITLE" | tr -d '\n' | sed 's/"/\\"/g') + echo "ISSUE_TITLE=$ISSUE_TITLE_SAFE" >> $GITHUB_OUTPUT + + # Generate unique branch name using timestamp and issue info + ISSUE_ID="${{ github.event.client_payload.issue_id }}" + BRANCH_NAME="aider-fix-linear-issue-$ISSUE_ID" + echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT + + - name: Probe Chat for Relevant Files + id: probe_files + env: + PROMPT_CONTENT_FILE: ${{ steps.create_prompt.outputs.PROMPT_FILE_PATH }} + run: | + echo "Running probe-chat to find relevant files..." + if [[ ! -f "$PROMPT_CONTENT_FILE" ]]; then + echo "::error::Prompt file $PROMPT_CONTENT_FILE not found!" + exit 1 + fi + PROMPT_CONTENT=$(cat "$PROMPT_CONTENT_FILE") + if [ -z "$PROMPT_CONTENT" ]; then + echo "::error::Prompt content is empty!" + exit 1 + fi + + PROMPT_ESCAPED=$(jq -Rs . <<< "$PROMPT_CONTENT") + + MESSAGE_FOR_PROBE=$(jq -n --arg prompt_escaped "$PROMPT_ESCAPED" \ + '{ "message": "I'\''m giving you a request that needs to be implemented. Your role is ONLY to give me the files that are relevant to the request and nothing else. The request is prepended with the word REQUEST.\\nREQUEST: \($prompt_escaped). Give me all the files relevant to this request. Your output MUST be a single json array that can be parsed with programatic json parsing, with the relevant files. Files can be rust or typescript or javascript files. DO NOT INCLUDE ANY OTHER TEXT IN YOUR OUTPUT. ONLY THE JSON ARRAY. Example of output: [\"file1.py\", \"file2.py\"]" }' | jq -r .message) + + set -o pipefail + PROBE_OUTPUT=$(npx --yes @buger/probe-chat@latest --max-iterations 50 --model-name gemini-2.5-pro-preview-05-06 --message "$MESSAGE_FOR_PROBE") || { + echo "::error::probe-chat command failed. Output:" + echo "$PROBE_OUTPUT" + exit 1 + } + set +o pipefail + echo "Probe-chat raw output:" + echo "$PROBE_OUTPUT" + + JSON_FILES=$(echo "$PROBE_OUTPUT" | sed -n '/^\s*\[/,$p' | sed '/^\s*\]/q') + echo "Extracted JSON block:" + echo "$JSON_FILES" + + FILES_LIST=$(echo "$JSON_FILES" | jq -e -r '[.[] | select(type == "string" and . != "" and . != null and (endswith("/") | not))] | map(@sh) | join(" ")' || echo "") + + if [[ -z "$FILES_LIST" ]]; then + echo "::warning::probe-chat did not identify any relevant files." + exit 1 + fi + + echo "Formatted files list for aider: $FILES_LIST" + echo "FILES_TO_EDIT=$FILES_LIST" >> $GITHUB_ENV + + - name: Run Aider with external prompt + run: | + echo "Files identified by probe-chat: ${{ env.FILES_TO_EDIT }}" + aider \ + --read .cursor/rules/rust-best-practices.mdc \ + --read .cursor/rules/svelte5-best-practices.mdc \ + --read .cursor/rules/windmill-overview.mdc \ + ${{ env.FILES_TO_EDIT }} \ + --model gemini/gemini-2.5-pro-preview-05-06 \ + --message-file .github/aider/issue-prompt.txt \ + --yes \ + --no-check-update \ + --auto-commits \ + --no-analytics \ + --no-gitignore \ + | tee .github/aider/aider-output.txt || true + echo "Aider command completed. Output saved to .github/aider/aider-output.txt" + + - name: Clean up prompt file + if: always() + run: rm -f .github/aider/issue-prompt.txt + + - name: Commit and Push Changes + id: commit_and_push + if: ${{ success() }} + run: | + BRANCH_NAME="${{ steps.create_prompt.outputs.BRANCH_NAME }}" + + # Check if branch exists remotely + if git ls-remote --heads origin $BRANCH_NAME | grep -q $BRANCH_NAME; then + echo "Branch $BRANCH_NAME already exists remotely, fetching it" + git fetch origin $BRANCH_NAME + git checkout $BRANCH_NAME + git pull origin $BRANCH_NAME + else + echo "Creating new branch $BRANCH_NAME" + git checkout -b $BRANCH_NAME + fi + + # Check if there are any changes to commit + if git diff --quiet && git diff --staged --quiet; then + echo "No changes to commit" + else + git commit -am "Auto-fix using Aider for external issue [skip ci]" || echo "No changes to commit" + fi + + git push origin $BRANCH_NAME + echo "Pushed to branch $BRANCH_NAME" + echo "PR_BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT + + - name: Create Pull Request + if: success() && steps.commit_and_push.outputs.PR_BRANCH_NAME != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_BRANCH: ${{ steps.commit_and_push.outputs.PR_BRANCH_NAME }} + ISSUE_TITLE: ${{ steps.create_prompt.outputs.ISSUE_TITLE }} + ISSUE_ID: ${{ github.event.client_payload.issue_id }} + run: | + # Create PR description in a temporary file to avoid command line length limits + cat > /tmp/pr-description.md << EOL + This PR was created automatically by Aider to fix an external issue: ${ISSUE_TITLE} + + ## Aider Output + \`\`\` + $(cat .github/aider/aider-output.txt || echo "No output available") + \`\`\` + EOL + + # Create PR using the file for the body content + gh pr create \ + --title "[Aider PR] Fix: ${ISSUE_TITLE}" \ + --body-file /tmp/pr-description.md \ + --head "$PR_BRANCH" \ + --base main || echo "PR already exists or couldn't be created"