diff --git a/.agents/skills/native-trigger/SKILL.md b/.agents/skills/native-trigger/SKILL.md index 026c1900bf..781e200d0c 100644 --- a/.agents/skills/native-trigger/SKILL.md +++ b/.agents/skills/native-trigger/SKILL.md @@ -1,3 +1,8 @@ +--- +name: native-trigger +description: Guidance for adding native trigger services to Windmill. Use when implementing or modifying native trigger integrations across the backend and frontend. +--- + # Skill: Adding Native Trigger Services This skill provides comprehensive guidance for adding new native trigger services to Windmill. Native triggers allow external services (like Nextcloud, Google Drive, etc.) to trigger Windmill scripts/flows via webhooks or push notifications. diff --git a/.claude/review-prompt.md b/.claude/review-prompt.md new file mode 100644 index 0000000000..6814089bea --- /dev/null +++ b/.claude/review-prompt.md @@ -0,0 +1,25 @@ +# Code Review Instructions + +Review this pull request and provide comprehensive feedback. + +## Focus Areas + +- **Code quality and best practices** — does the code follow established patterns? +- **Potential bugs or issues** — will this code work correctly in all cases? +- **Performance considerations** — are there unnecessary allocations, N+1 queries, or bottlenecks? +- **Security implications** — injection, auth bypass, data exposure? + +## CLAUDE.md Compliance + +Read all relevant CLAUDE.md files (root and in directories containing changed files). Check each rule against the changed code. Quote the exact rule when flagging a violation. + +## Review Guidelines + +- Provide detailed feedback using inline comments for specific issues +- Use top-level comments for general observations or praise +- Only flag issues introduced by this PR, not pre-existing problems +- Self-validate each finding: "Is this definitely a real issue?" If uncertain, discard it + +## Testing Instructions + +At the end of your review, add complete instructions to reproduce the added changes through the app interface. These instructions will be given to a tester so they can verify the changes. It should be a short descriptive text (not a step-by-step or a list) on how to navigate the app (what page, what action, what input, etc.) to see the changes. diff --git a/.claude/skills/local-review/SKILL.md b/.claude/skills/local-review/SKILL.md index f14f5608db..0399ad7294 100644 --- a/.claude/skills/local-review/SKILL.md +++ b/.claude/skills/local-review/SKILL.md @@ -6,53 +6,24 @@ description: Code review a pull request for bugs and CLAUDE.md compliance. MUST # Local Code Review Skill -Review a pull request for real bugs and CLAUDE.md compliance violations. This review targets HIGH SIGNAL issues only. - -## Review Philosophy - -- **Only flag issues you are certain about.** If you are not sure an issue is real, do not flag it. False positives erode trust and waste reviewer time. -- Think like a senior engineer doing a final review — flag things that would cause incidents, not things that are merely imperfect. - -## What to Flag - -- Code that won't compile or parse (syntax errors, type errors, missing imports) -- Code that will definitely produce wrong results regardless of inputs -- Clear, unambiguous CLAUDE.md violations (quote the exact rule being violated) -- Security issues in introduced code (injection, auth bypass, data exposure) -- Incorrect logic that will fail in production - -## What NOT to Flag - -- Code style or quality concerns -- Potential issues that depend on specific inputs or runtime state -- Subjective suggestions or improvements -- Pre-existing issues not introduced by this PR -- Pedantic nitpicks a senior engineer wouldn't flag -- Issues a linter or type checker will catch -- General quality concerns unless explicitly prohibited in CLAUDE.md -- Issues silenced via lint ignore comments +Run the same review locally that the GitHub Claude Auto Review action runs on PRs. The shared review instructions live in `.claude/review-prompt.md` — read that file first and follow its instructions. ## Execution Steps -1. **Determine the PR scope**: +1. **Read `.claude/review-prompt.md`** for the review criteria and focus areas + +2. **Determine the PR scope**: - If an argument is provided, use it as the PR number or branch - Otherwise, detect from the current branch vs main - Run `gh pr view` if a PR exists, or use `git diff main...HEAD` -2. **Find relevant CLAUDE.md files**: - - Read the root `CLAUDE.md` - - Check for CLAUDE.md files in directories containing changed files - 3. **Get the diff and metadata**: - `gh pr diff` or `git diff main...HEAD` for the full diff - `gh pr view` or `git log main..HEAD --oneline` for context 4. **Read changed files** where the diff alone is insufficient to understand context -5. **Review for**: - - CLAUDE.md compliance — check each rule against the changed code - - Bugs and logic errors — will this code work correctly? - - Security issues — injection, auth, data exposure in new code +5. **Apply the review instructions from `.claude/review-prompt.md`** 6. **Self-validate each finding**: Before reporting, ask yourself: - "Is this definitely a real issue, not a false positive?" diff --git a/.claude/skills/native-trigger/SKILL.md b/.claude/skills/native-trigger/SKILL.md index 026c1900bf..781e200d0c 100644 --- a/.claude/skills/native-trigger/SKILL.md +++ b/.claude/skills/native-trigger/SKILL.md @@ -1,3 +1,8 @@ +--- +name: native-trigger +description: Guidance for adding native trigger services to Windmill. Use when implementing or modifying native trigger integrations across the backend and frontend. +--- + # Skill: Adding Native Trigger Services This skill provides comprehensive guidance for adding new native trigger services to Windmill. Native triggers allow external services (like Nextcloud, Google Drive, etc.) to trigger Windmill scripts/flows via webhooks or push notifications. diff --git a/.claude/skills/pr/SKILL.md b/.claude/skills/pr/SKILL.md index ab67b58748..2c7bd691ca 100644 --- a/.claude/skills/pr/SKILL.md +++ b/.claude/skills/pr/SKILL.md @@ -61,12 +61,13 @@ Generated with [Claude Code](https://claude.com/claude-code) 1. Run `git status` to check for uncommitted changes 2. Run `git log main..HEAD --oneline` to see all commits in this branch 3. Run `git diff main...HEAD` to see the full diff against main -4. Check if remote branch exists and is up to date: +4. **Run `/local-review`** before creating the PR. If issues are found, fix them and commit before proceeding. Do not skip this step. +5. Check if remote branch exists and is up to date: ```bash git rev-parse --abbrev-ref --symbolic-full-name @{u} 2>/dev/null || echo "no upstream" ``` -5. Push to remote if needed: `git push -u origin HEAD` -6. Create draft PR using gh CLI: +6. Push to remote if needed: `git push -u origin HEAD` +7. Create draft PR using gh CLI: ```bash gh pr create --draft --title ": " --body "$(cat <<'EOF' ## Summary @@ -85,7 +86,7 @@ Generated with [Claude Code](https://claude.com/claude-code) EOF )" ``` -7. Return the PR URL to the user +8. Return the PR URL to the user ## EE Companion PR (when `*_ee.rs` files were modified) diff --git a/.github/codex/pr-review.prompt.md b/.github/codex/pr-review.prompt.md new file mode 100644 index 0000000000..d3e6dfc4e8 --- /dev/null +++ b/.github/codex/pr-review.prompt.md @@ -0,0 +1,23 @@ +You are reviewing a GitHub pull request for this repository. + +Review policy: +- Read `CLAUDE.md` before reviewing code. +- Only report issues you are confident are real and introduced by this pull request. +- Focus on bugs, security problems, and clear `CLAUDE.md` violations. +- Do not report style nits, speculative concerns, pre-existing issues, or problems that a normal linter/typechecker would obviously catch. +- Keep the review high signal. If there is no clear issue, return no findings. + +Repository context: +- Read `./.github/codex/pr-review-context.md` for the PR metadata and the exact diff commands to use. +- Review only the changes introduced by this PR. +- Read additional files only when the diff is not enough to validate a finding. +- Do not modify any files. + +Output requirements: +- Return a GitHub PR comment in markdown, not JSON. +- Start with `## Codex Review`. +- Give a short overall summary first. +- If you found high-signal issues, list them in a short numbered list with file paths and line numbers when you know them confidently. +- If you found no high-signal issues, say that explicitly. +- End with a `### Reproduction instructions` section containing a short descriptive paragraph for a tester explaining how to navigate the app to observe the change. Do not make it a numbered list. If the diff is not enough to infer this safely, say that plainly. +- Prefer at most 10 findings. diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 195821b2dd..d420ff1f00 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -290,6 +290,49 @@ jobs: path: | *.json + benchmark_wac: + runs-on: ubicloud-standard-8 + services: + postgres: + image: postgres + env: + POSTGRES_DB: windmill + POSTGRES_PASSWORD: changeme + POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB" + options: >- + --health-cmd pg_isready --health-interval 10s --health-timeout 5s + --health-retries 5 + --shm-size=2g + windmill: + image: ghcr.io/windmill-labs/windmill-ee:main + env: + DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill + LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} + WORKER_GROUP: main + WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + options: >- + --pull always --health-interval 10s --health-timeout 5s + --health-retries 5 --health-cmd "curl + http://localhost:8000/api/version" + ports: + - 8000:8000 + steps: + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + - name: benchmark + timeout-minutes: 30 + run: deno run -A -r + https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts + -c + https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_wac.json + - name: Save benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark_wac + path: | + *.json + benchmark_graphs: runs-on: ubicloud needs: @@ -297,6 +340,7 @@ jobs: - benchmark_dedicated - benchmark_4workers - benchmark_8workers + - benchmark_wac steps: - uses: denoland/setup-deno@v2 with: diff --git a/.github/workflows/cli-tests.yml b/.github/workflows/cli-tests.yml index 9c87a249a3..237a5ff555 100644 --- a/.github/workflows/cli-tests.yml +++ b/.github/workflows/cli-tests.yml @@ -1,6 +1,7 @@ name: CLI Tests on: + workflow_dispatch: push: branches: [main] paths: diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml new file mode 100644 index 0000000000..e945f5fd45 --- /dev/null +++ b/.github/workflows/codex-pr-review.yml @@ -0,0 +1,145 @@ +name: Codex Auto Review + +on: + pull_request: + types: [ready_for_review, opened] + +concurrency: + group: codex-review-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + codex-review: + runs-on: ubicloud-standard-2 + timeout-minutes: 30 + if: github.event.pull_request.draft == false && github.event.pull_request.head.repo.fork == false + permissions: + contents: read + issues: write + steps: + - name: Check Codex configuration + id: codex_config + env: + CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }} + run: | + if [ -n "$CODEX_AUTH_JSON" ]; then + echo "enabled=true" >> "$GITHUB_OUTPUT" + else + echo "enabled=false" >> "$GITHUB_OUTPUT" + echo "CODEX_AUTH_JSON is not configured; skipping Codex review." + fi + + - name: Checkout repository + if: steps.codex_config.outputs.enabled == 'true' + uses: actions/checkout@v5 + with: + ref: refs/pull/${{ github.event.pull_request.number }}/merge + fetch-depth: 1 + + - name: Set up Node.js + if: steps.codex_config.outputs.enabled == 'true' + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install Codex CLI + if: steps.codex_config.outputs.enabled == 'true' + run: npm install --global @openai/codex@0.117.0 + + - name: Configure file-backed Codex auth + if: steps.codex_config.outputs.enabled == 'true' + env: + CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }} + run: | + CODEX_HOME="$HOME/.codex" + echo "CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV" + mkdir -p "$CODEX_HOME" + chmod 700 "$CODEX_HOME" + cat > "$CODEX_HOME/config.toml" <<'EOF' + cli_auth_credentials_store = "file" + EOF + printf '%s' "$CODEX_AUTH_JSON" > "$CODEX_HOME/auth.json" + chmod 600 "$CODEX_HOME/auth.json" + node -e 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"))' "$CODEX_HOME/auth.json" + + - name: Pre-fetch base and head refs for the PR + if: steps.codex_config.outputs.enabled == 'true' + env: + PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + git fetch --no-tags origin \ + "$PR_BASE_REF" \ + "+refs/pull/$PR_NUMBER/head" + + - name: Write Codex review context + if: steps.codex_config.outputs.enabled == 'true' + env: + PR_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_TITLE: ${{ github.event.pull_request.title }} + PR_BODY: ${{ github.event.pull_request.body || '' }} + run: | + mkdir -p .github/codex + node <<'NODE' + const fs = require('fs'); + const lines = [ + `Repository: ${process.env.PR_REPOSITORY}`, + `PR number: ${process.env.PR_NUMBER}`, + `Base SHA: ${process.env.PR_BASE_SHA}`, + `Head SHA: ${process.env.PR_HEAD_SHA}`, + '', + 'PR title:', + process.env.PR_TITLE || '(empty)', + '', + 'PR body:', + process.env.PR_BODY || '(empty)', + '', + 'Changed commits command:', + `git log --oneline ${process.env.PR_BASE_SHA}...${process.env.PR_HEAD_SHA}`, + '', + 'Changed files command:', + `git diff --stat ${process.env.PR_BASE_SHA}...${process.env.PR_HEAD_SHA}`, + '', + 'Full review diff command:', + `git diff --unified=0 ${process.env.PR_BASE_SHA}...${process.env.PR_HEAD_SHA}` + ]; + fs.writeFileSync('.github/codex/pr-review-context.md', `${lines.join('\n')}\n`); + NODE + + - name: Run Codex review + if: steps.codex_config.outputs.enabled == 'true' + run: | + codex exec \ + -C "$GITHUB_WORKSPACE" \ + -m gpt-5.4 \ + -c 'model_reasoning_effort="xhigh"' \ + -s read-only \ + -o codex-final-message.md \ + - < .github/codex/pr-review.prompt.md + + - name: Post Codex review comment + if: steps.codex_config.outputs.enabled == 'true' + uses: actions/github-script@v7 + with: + github-token: ${{ github.token }} + script: | + const fs = require('fs'); + const path = `${process.env.GITHUB_WORKSPACE}/codex-final-message.md`; + if (!fs.existsSync(path)) { + core.info('Codex did not produce a final message; skipping PR comment.'); + return; + } + const body = fs.readFileSync(path, 'utf8').trim(); + if (!body) { + core.info('Codex final message was empty; skipping PR comment.'); + return; + } + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + body, + }); diff --git a/.github/workflows/pr-ready-review.yml b/.github/workflows/pr-ready-review.yml index 25553bcb2c..78c0c3e045 100644 --- a/.github/workflows/pr-ready-review.yml +++ b/.github/workflows/pr-ready-review.yml @@ -22,6 +22,15 @@ jobs: with: fetch-depth: 1 + - name: Read review prompt + id: review-prompt + run: | + { + echo 'REVIEW_PROMPT<> "$GITHUB_ENV" + - name: Automatic PR Review uses: anthropics/claude-code-action@v1 with: @@ -31,18 +40,7 @@ jobs: REPO: ${{ github.repository }} PR NUMBER: ${{ github.event.pull_request.number }} - Please review this pull request and provide comprehensive feedback. - - Focus on: - - Code quality and best practices - - Potential bugs or issues - - Performance considerations - - Security implications - - Provide detailed feedback using inline comments for specific issues. - Use top-level comments for general observations or praise. - - At the end of your review, add complete instructions to reproduce the added changes through the app interface. These instructions will be given to a tester so he can verify the changes. It should be a short descriptive text (not a step by step or a list) on how to navigate the app (what page, what action, what input, etc) to see the changes. + ${{ env.REVIEW_PROMPT }} claude_args: | --allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*)" --model opus diff --git a/.webmux.yaml b/.webmux.yaml index c41d0aa699..e00435465d 100644 --- a/.webmux.yaml +++ b/.webmux.yaml @@ -43,7 +43,7 @@ profiles: - Pane 0: this pane (claude agent) - Pane 1: backend (cargo watch -x run) - Pane 2: frontend (npm run dev) - To check logs, use: \`tmux capture-pane -t .1 -p -S -50\` (backend) or \`tmux capture-pane -t .2 -p -S -50\` (frontend). + To check logs, use: \`tmux capture-pane -t $(tmux display-message -t "$TMUX_PANE" -p '#{session_name}:#{window_name}').1 -p -S -50\` (backend) or \`tmux capture-pane -t $(tmux display-message -t "$TMUX_PANE" -p '#{session_name}:#{window_name}').2 -p -S -50\` (frontend). For this window specifically, backend is running on: ${BACKEND_PORT} and frontend is running on: ${FRONTEND_PORT}. To connect to the database, use this connection string: ${DATABASE_URL} Because we are running backend with cargo watch, to verify your changes, just check the logs in the backend pane. No need for cargo check. @@ -72,7 +72,7 @@ profiles: Pane layout (current window): - Pane 0: this pane (claude agent) - Pane 1: frontend (npm run dev) - To check logs, use: \`tmux capture-pane -t .1 -p -S -50\` (frontend). + To check logs, use: \`tmux capture-pane -t $(tmux display-message -t "$TMUX_PANE" -p '#{session_name}:#{window_name}').1 -p -S -50\` (frontend). On this window specifically, frontend is running on: ${FRONTEND_PORT}. To connect to the database, use this connection string: ${DATABASE_URL} Because we are running frontend with npm run dev, to verify your changes, just check the logs in the frontend pane. No need for npm run build. diff --git a/CHANGELOG.md b/CHANGELOG.md index f9d6640b9f..9e47211680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,181 @@ # Changelog +## [1.672.0](https://github.com/windmill-labs/windmill/compare/v1.671.0...v1.672.0) (2026-04-01) + + +### Features + +* add R language support ([#8263](https://github.com/windmill-labs/windmill/issues/8263)) ([a46aa64](https://github.com/windmill-labs/windmill/commit/a46aa641f9d72809c52a0eb11a877a0f2d587c32)) + + +### Bug Fixes + +* approval page freeze, stale state, and missing approval link ([#8653](https://github.com/windmill-labs/windmill/issues/8653)) ([7069202](https://github.com/windmill-labs/windmill/commit/70692021909443b86ed61fa621fe49f28742fb54)) + +## [1.671.0](https://github.com/windmill-labs/windmill/compare/v1.670.0...v1.671.0) (2026-03-31) + + +### Features + +* add configurable preview job tag override in default tags settings ([#8649](https://github.com/windmill-labs/windmill/issues/8649)) ([da8886b](https://github.com/windmill-labs/windmill/commit/da8886be8575dd925b6d24c55ab379bc6984c5f8)) +* improve CLI flow log streaming and job inspection ([#8644](https://github.com/windmill-labs/windmill/issues/8644)) ([6c3c971](https://github.com/windmill-labs/windmill/commit/6c3c971af5aa1362632ee0deeddf91b8bc47c853)) +* support hub flows in raw app runnables ([#8627](https://github.com/windmill-labs/windmill/issues/8627)) ([040a199](https://github.com/windmill-labs/windmill/commit/040a199685cea5c99c944bacb5584a381d6ec829)) + + +### Bug Fixes + +* return default_args/enums in approval info and fix subflow resume buttons ([#8648](https://github.com/windmill-labs/windmill/issues/8648)) ([852c59e](https://github.com/windmill-labs/windmill/commit/852c59efbb04510e5e6f99919707effcf6769a2f)) + +## [1.670.0](https://github.com/windmill-labs/windmill/compare/v1.669.1...v1.670.0) (2026-03-31) + + +### Features + +* add OR logic support to kafka/websocket trigger filters ([#8580](https://github.com/windmill-labs/windmill/issues/8580)) ([3876902](https://github.com/windmill-labs/windmill/commit/3876902a7be798fd5ef208bc5756b28fb55e569e)) +* expose getJob and getJobLogs as MCP tools ([#8632](https://github.com/windmill-labs/windmill/issues/8632)) ([cd8edcd](https://github.com/windmill-labs/windmill/commit/cd8edcd94f2bf44c3e771000cb0bbad08accc0e7)) +* support multiline secrets in resource password fields ([#8637](https://github.com/windmill-labs/windmill/issues/8637)) ([26050f9](https://github.com/windmill-labs/windmill/commit/26050f96c34f14826298760174a45f3559d3266c)) +* support sensitive/secret fields for non-string types ([#8635](https://github.com/windmill-labs/windmill/issues/8635)) ([375fb66](https://github.com/windmill-labs/windmill/commit/375fb66abe2d1861b53dc2b36d2cf0e2eb82c3a8)) + + +### Bug Fixes + +* cap input history per_page to 100 on cloud ([#8624](https://github.com/windmill-labs/windmill/issues/8624)) ([8e973c8](https://github.com/windmill-labs/windmill/commit/8e973c892d768be2da2e6b4b7af9e40b62333052)) +* compute highest workspace role across all instance groups ([#8633](https://github.com/windmill-labs/windmill/issues/8633)) ([92b9ac7](https://github.com/windmill-labs/windmill/commit/92b9ac72c5fc9a5085fcb2e9d835ccbb53bcd4b0)) +* Ducklake UI Nits ([#8628](https://github.com/windmill-labs/windmill/issues/8628)) ([ef1757f](https://github.com/windmill-labs/windmill/commit/ef1757f5d747e513d201eb6fa48918dba8248abe)) +* preserve flow notes/groups and field ordering in generate-metadata ([#8641](https://github.com/windmill-labs/windmill/issues/8641)) ([#8642](https://github.com/windmill-labs/windmill/issues/8642)) ([52a04d2](https://github.com/windmill-labs/windmill/commit/52a04d210f476f4598007f67770bc6520b045950)) +* remove timeout on python client httpx to prevent ducklake query timeouts ([#8636](https://github.com/windmill-labs/windmill/issues/8636)) ([c5fccd2](https://github.com/windmill-labs/windmill/commit/c5fccd2f69ad8a6e46c514cf89b9aa21b380e6fe)) +* resolve missing form schema for nested suspend steps in FlowNode sub-flows ([#8643](https://github.com/windmill-labs/windmill/issues/8643)) ([12ea7e7](https://github.com/windmill-labs/windmill/commit/12ea7e74237560a9dfc99b6bc1338e3343b57640)) +* smarter secret masking based on secret length ([#8629](https://github.com/windmill-labs/windmill/issues/8629)) ([bfc2aef](https://github.com/windmill-labs/windmill/commit/bfc2aefdb8ab92b7284de7f9e485a5504502d944)) + +## [1.669.1](https://github.com/windmill-labs/windmill/compare/v1.669.0...v1.669.1) (2026-03-30) + + +### Bug Fixes + +* avoid doubled /oauth2 path in Okta custom authorization server URLs ([#8620](https://github.com/windmill-labs/windmill/issues/8620)) ([4817913](https://github.com/windmill-labs/windmill/commit/4817913f0cab49980bfeb442089631d7953955ff)) +* improve db health UI text and prevent label wrapping ([d532c1d](https://github.com/windmill-labs/windmill/commit/d532c1d470fcb0ef02ebc5342ad1cf22e58b1f4d)) + +## [1.669.0](https://github.com/windmill-labs/windmill/compare/v1.668.5...v1.669.0) (2026-03-30) + + +### Features + +* WAC workflow diagram visualization via WASM ([#8604](https://github.com/windmill-labs/windmill/issues/8604)) ([abc6b12](https://github.com/windmill-labs/windmill/commit/abc6b12d6815edc4dda3ddf5f0572ecedcb670dd)) + + +### Bug Fixes + +* add path traversal check in service_logs get_log_file endpoint ([#8605](https://github.com/windmill-labs/windmill/issues/8605)) ([5f2d3e6](https://github.com/windmill-labs/windmill/commit/5f2d3e6812f01fe6194bcfd976970a6e3c4186cc)) +* cast DuckDB IS_NULLABLE to string in metadata query ([#8607](https://github.com/windmill-labs/windmill/issues/8607)) ([f3012ee](https://github.com/windmill-labs/windmill/commit/f3012ee7ccc7a8947b5f6bd7c7df77984437f91e)) +* enable S3 bundle cache for PHP previews without lock file ([#8608](https://github.com/windmill-labs/windmill/issues/8608)) ([ee62315](https://github.com/windmill-labs/windmill/commit/ee6231590ed91063f104e6d054b52e88b569986f)) +* enforce workspace isolation on flow resume endpoint ([#8612](https://github.com/windmill-labs/windmill/issues/8612)) ([33032ed](https://github.com/windmill-labs/windmill/commit/33032ed297cf9ea867388d4ea2ece607c9d36dc7)) +* handle DuckDB boolean types in ColumnDef deserializers ([#8610](https://github.com/windmill-labs/windmill/issues/8610)) ([22da5bd](https://github.com/windmill-labs/windmill/commit/22da5bd9ea1ca000cfab3eecf1e3fb0fc01200cb)) +* use route_service instead of fallback_service for MCP router ([#8614](https://github.com/windmill-labs/windmill/issues/8614)) ([98934d5](https://github.com/windmill-labs/windmill/commit/98934d59c552325fcf88c016e31ae977970e8c9a)) + +## [1.668.5](https://github.com/windmill-labs/windmill/compare/v1.668.4...v1.668.5) (2026-03-29) + + +### Bug Fixes + +* add per-IP and per-account brute force protection on login endpoint ([#8601](https://github.com/windmill-labs/windmill/issues/8601)) ([06bbe7b](https://github.com/windmill-labs/windmill/commit/06bbe7b94bfb846bd73aaf6abdc83e4c14e70adc)) +* add timestamp validation to webhook signature verification ([#8596](https://github.com/windmill-labs/windmill/issues/8596)) ([74fba2a](https://github.com/windmill-labs/windmill/commit/74fba2abf3dc68b682777c01da360258786fded8)) +* disable workspace webhook events when CLOUD_HOSTED ([#8598](https://github.com/windmill-labs/windmill/issues/8598)) ([be7fbeb](https://github.com/windmill-labs/windmill/commit/be7fbeb8b1f31d15e33b0783b2a504d6a01e532e)) +* harden login rate limiting with CLOUD_HOSTED gating and memory eviction ([#8602](https://github.com/windmill-labs/windmill/issues/8602)) ([754b88a](https://github.com/windmill-labs/windmill/commit/754b88a52c4e76421cb21c1eed87ad9d8385e9aa)) +* prevent SSRF and local file read via git repository resource URLs ([#8600](https://github.com/windmill-labs/windmill/issues/8600)) ([845db72](https://github.com/windmill-labs/windmill/commit/845db72b7344fb87ac9c5e24697750549665c7bf)) +* rename snippet param to avoid svelte compiler shadowing bug in asset usages drawer ([#8595](https://github.com/windmill-labs/windmill/issues/8595)) ([8c770a2](https://github.com/windmill-labs/windmill/commit/8c770a206a3b0704642c0bda2ab2aeb199d8af3f)) +* require mcp: scope for MCP endpoints instead of blanket bypass ([#8597](https://github.com/windmill-labs/windmill/issues/8597)) ([f5fc9f8](https://github.com/windmill-labs/windmill/commit/f5fc9f8485d2ec3e20f8b451305195446b90e5a3)) +* use constant-time comparison for API key and basic auth validation ([#8593](https://github.com/windmill-labs/windmill/issues/8593)) ([b4d1f2a](https://github.com/windmill-labs/windmill/commit/b4d1f2aac789306c2e35e123ac93e12c47c26f99)) +* validate JSON before sql_builder bind to prevent injection via JSONB queries ([#8599](https://github.com/windmill-labs/windmill/issues/8599)) ([970e859](https://github.com/windmill-labs/windmill/commit/970e859a410b0144847a1a30d7059955effdd402)) + +## [1.668.4](https://github.com/windmill-labs/windmill/compare/v1.668.3...v1.668.4) (2026-03-29) + + +### Bug Fixes + +* update git sync version to latest cli ([0549f68](https://github.com/windmill-labs/windmill/commit/0549f682fe14f4d4b2f67941362ed2cc29d974a1)) + +## [1.668.3](https://github.com/windmill-labs/windmill/compare/v1.668.2...v1.668.3) (2026-03-28) + + +### Bug Fixes + +* **cli:** phantom diffs, flow safety, trigger DX, lint watch, error clarity ([#8588](https://github.com/windmill-labs/windmill/issues/8588)) ([c6ce319](https://github.com/windmill-labs/windmill/commit/c6ce3197a72ceeffd702cf2263b1074ecbf1ca33)) + +## [1.668.2](https://github.com/windmill-labs/windmill/compare/v1.668.1...v1.668.2) (2026-03-28) + + +### Bug Fixes + +* **cli:** app push crash, lint path, push --message, run validation, history timestamps ([#8585](https://github.com/windmill-labs/windmill/issues/8585)) ([f40cdaf](https://github.com/windmill-labs/windmill/commit/f40cdaf43453d2643800ed730d6abe6873bbe8e7)) + +## [1.668.1](https://github.com/windmill-labs/windmill/compare/v1.668.0...v1.668.1) (2026-03-28) + + +### Bug Fixes + +* **cli:** fix 13 CLI bugs — exit codes, sync tar fallback, variable encryption, JSON output ([#8582](https://github.com/windmill-labs/windmill/issues/8582)) ([38acaa3](https://github.com/windmill-labs/windmill/commit/38acaa3653728bf9e0ae6f746edf433703b4ab63)) + +## [1.668.0](https://github.com/windmill-labs/windmill/compare/v1.667.0...v1.668.0) (2026-03-28) + + +### Features + +* add DB health diagnostic dashboard for superadmins ([#8574](https://github.com/windmill-labs/windmill/issues/8574)) ([9ceab73](https://github.com/windmill-labs/windmill/commit/9ceab730d7def09c2b46527f8a586789d14f2ce0)) +* **cli:** add job, group, audit, token commands and schedule enable/disable ([#8581](https://github.com/windmill-labs/windmill/issues/8581)) ([d29cb23](https://github.com/windmill-labs/windmill/commit/d29cb234dbff07473b911e5e75e362def8a47650)) +* IAM RDS auth for PostgreSQL worker resources ([#8573](https://github.com/windmill-labs/windmill/issues/8573)) ([56253c0](https://github.com/windmill-labs/windmill/commit/56253c04cb679c58d00750da699a6cb62ed52aca)) + + +### Bug Fixes + +* add Authority Key Identifier to MITM proxy leaf certs ([#8576](https://github.com/windmill-labs/windmill/issues/8576)) ([ce2e6c8](https://github.com/windmill-labs/windmill/commit/ce2e6c8c015110d0385e6afecdc8313aabca1364)) +* Improve CLI developer experience: error handling, sync workflow, JSON output, workspace forks ([#8578](https://github.com/windmill-labs/windmill/issues/8578)) ([501a4ff](https://github.com/windmill-labs/windmill/commit/501a4ff2a94510145952686d24ccc639781beefe)) +* trigger capture filter and focus issues ([#8579](https://github.com/windmill-labs/windmill/issues/8579)) ([820f28f](https://github.com/windmill-labs/windmill/commit/820f28f8799f8dad5cfab94b51ac9921d664f04a)) + +## [1.667.0](https://github.com/windmill-labs/windmill/compare/v1.666.0...v1.667.0) (2026-03-27) + + +### Features + +* add schedule support to CLI branch-specific items ([#8570](https://github.com/windmill-labs/windmill/issues/8570)) ([b592996](https://github.com/windmill-labs/windmill/commit/b592996eee98ddb664f1b007b95a2096d5d4e3a6)) +* add workspace-level service accounts ([#8560](https://github.com/windmill-labs/windmill/issues/8560)) ([3959fe8](https://github.com/windmill-labs/windmill/commit/3959fe82974f5f0383e94fd83a5d78fe4212d56a)) +* **cli:** generate commented wmill.yaml and add config reference command ([#8546](https://github.com/windmill-labs/windmill/issues/8546)) ([d06b426](https://github.com/windmill-labs/windmill/commit/d06b42613f73c4a7b31c990be22b0c97efab2666)) +* DB-coordinated graceful restart staggering for settings changes ([#8555](https://github.com/windmill-labs/windmill/issues/8555)) ([2f32675](https://github.com/windmill-labs/windmill/commit/2f326758013dd1f1e6ae732e5784a32f1fb6e4bd)) +* improve-replay-ui ([#8250](https://github.com/windmill-labs/windmill/issues/8250)) ([c0aafee](https://github.com/windmill-labs/windmill/commit/c0aafee9a9923d5dc2fa3b99da4378e923933a06)) +* support multiple folder selection in MCP scope selector ([#8557](https://github.com/windmill-labs/windmill/issues/8557)) ([ad19ac9](https://github.com/windmill-labs/windmill/commit/ad19ac9b37b04591c921f93f180bdda961af6cef)) + + +### Bug Fixes + +* **cli:** preserve inline script files during flow generate-locks ([#8561](https://github.com/windmill-labs/windmill/issues/8561)) ([a8b651d](https://github.com/windmill-labs/windmill/commit/a8b651da9ff86766119e14c0b61652be8a7b453a)) +* emit 0 for OTEL queue metrics when tag queue is empty ([#8559](https://github.com/windmill-labs/windmill/issues/8559)) ([79cc4a9](https://github.com/windmill-labs/windmill/commit/79cc4a92d88486c999799826bd0c9663767103f5)) +* handle inline script deletion in sync push + flow new nonDottedPaths ([#8553](https://github.com/windmill-labs/windmill/issues/8553)) ([943fe9c](https://github.com/windmill-labs/windmill/commit/943fe9c6cc9b046e24007e45b5c37afc4804256a)) +* include importer_kind in dependency debounce key to prevent cross-kind collisions ([#8567](https://github.com/windmill-labs/windmill/issues/8567)) ([bc7007b](https://github.com/windmill-labs/windmill/commit/bc7007bb4265e1f1375c1f0678b74325882a4e92)) +* multi-script dedicated workers race on shared job_dir ([#8551](https://github.com/windmill-labs/windmill/issues/8551)) ([#8569](https://github.com/windmill-labs/windmill/issues/8569)) ([63a3573](https://github.com/windmill-labs/windmill/commit/63a3573951d1f724cc63728ed973d039a5468072)) +* preserve notes on nodes inside collapsed groups ([#8552](https://github.com/windmill-labs/windmill/issues/8552)) ([0fb1153](https://github.com/windmill-labs/windmill/commit/0fb115304afc49812420e9ce24e5048502621059)) +* sanitize flow step summaries for filesystem-safe names ([#8554](https://github.com/windmill-labs/windmill/issues/8554)) ([e15bfbf](https://github.com/windmill-labs/windmill/commit/e15bfbf91ee1517432a6861ebb48e129485006aa)) +* use admin db pool in get_copilot_settings_state ([#8564](https://github.com/windmill-labs/windmill/issues/8564)) ([70f3ee5](https://github.com/windmill-labs/windmill/commit/70f3ee5ed4470e9993be822874f2b38e83a96611)) + + +### Performance Improvements + +* enable bun bundle caching for WAC v2 scripts ([#8556](https://github.com/windmill-labs/windmill/issues/8556)) ([ab868e9](https://github.com/windmill-labs/windmill/commit/ab868e9ebceadaa55e54770d9d59dc5524da13ff)) + +## [1.666.0](https://github.com/windmill-labs/windmill/compare/v1.665.0...v1.666.0) (2026-03-26) + + +### Features + +* add PDF input support to AI agent ([#8525](https://github.com/windmill-labs/windmill/issues/8525)) ([e44504c](https://github.com/windmill-labs/windmill/commit/e44504c6e93e7a4ee94ced03ab626b79a4fd0754)) + + +### Bug Fixes + +* add relative imports to the dependency list in deploymentUI ([#8548](https://github.com/windmill-labs/windmill/issues/8548)) ([d760ea5](https://github.com/windmill-labs/windmill/commit/d760ea5eaf4dc33007f1fd3e5e07b86925a0aa11)) +* filter null entries in FileUpload initialValue to prevent s3 access error ([#8544](https://github.com/windmill-labs/windmill/issues/8544)) ([1a73012](https://github.com/windmill-labs/windmill/commit/1a73012e0737a6ebea8307013dc0f79982269d91)) +* pass pre-bound TcpListener to run_server to fix Windows CI test race ([#8542](https://github.com/windmill-labs/windmill/issues/8542)) ([d7f4b95](https://github.com/windmill-labs/windmill/commit/d7f4b950ce6e966ed1b410e03d48fe96bc036e73)) +* resolve parent_hash race condition in sync push with auto_parent ([#8545](https://github.com/windmill-labs/windmill/issues/8545)) ([71549c3](https://github.com/windmill-labs/windmill/commit/71549c3db053bcc209c7065ac8cd42f1e8047cc3)) +* upload_s3_file not working in VS Code extension ([#8547](https://github.com/windmill-labs/windmill/issues/8547)) ([1fa4d91](https://github.com/windmill-labs/windmill/commit/1fa4d919b30ac9eff2d1789fba2695450ba115e7)) + ## [1.665.0](https://github.com/windmill-labs/windmill/compare/v1.664.0...v1.665.0) (2026-03-26) diff --git a/backend/.sqlx/query-077467cd813d5af161cb1cc232724f26984822d4c28ba36c0a9331273b10edc0.json b/backend/.sqlx/query-077467cd813d5af161cb1cc232724f26984822d4c28ba36c0a9331273b10edc0.json new file mode 100644 index 0000000000..0f5113bac5 --- /dev/null +++ b/backend/.sqlx/query-077467cd813d5af161cb1cc232724f26984822d4c28ba36c0a9331273b10edc0.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO email_to_igroup (email, igroup) VALUES ('alice@example.com', 'admins') ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "077467cd813d5af161cb1cc232724f26984822d4c28ba36c0a9331273b10edc0" +} diff --git a/backend/.sqlx/query-07770a002a49428c4f956cfc7262d6b6792ae5b97ed90b0ee07d17480b2dffe2.json b/backend/.sqlx/query-07770a002a49428c4f956cfc7262d6b6792ae5b97ed90b0ee07d17480b2dffe2.json new file mode 100644 index 0000000000..145fdc1229 --- /dev/null +++ b/backend/.sqlx/query-07770a002a49428c4f956cfc7262d6b6792ae5b97ed90b0ee07d17480b2dffe2.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT setting::bigint as \"max!\" FROM pg_settings WHERE name = 'max_connections'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "07770a002a49428c4f956cfc7262d6b6792ae5b97ed90b0ee07d17480b2dffe2" +} diff --git a/backend/.sqlx/query-143acebe5d815c5d828013ebe46274f891f953c75f821499552ab7794f75063d.json b/backend/.sqlx/query-143acebe5d815c5d828013ebe46274f891f953c75f821499552ab7794f75063d.json new file mode 100644 index 0000000000..3514370ec8 --- /dev/null +++ b/backend/.sqlx/query-143acebe5d815c5d828013ebe46274f891f953c75f821499552ab7794f75063d.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements') as \"exists!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "143acebe5d815c5d828013ebe46274f891f953c75f821499552ab7794f75063d" +} diff --git a/backend/.sqlx/query-1721f8b52ea265c0537fd7c742deddf0afbe5cf0d81b15e487c411ae169d3a89.json b/backend/.sqlx/query-1721f8b52ea265c0537fd7c742deddf0afbe5cf0d81b15e487c411ae169d3a89.json new file mode 100644 index 0000000000..ba4ad17175 --- /dev/null +++ b/backend/.sqlx/query-1721f8b52ea265c0537fd7c742deddf0afbe5cf0d81b15e487c411ae169d3a89.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT igroup FROM email_to_igroup WHERE email = 'alice@example.com'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "igroup", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "1721f8b52ea265c0537fd7c742deddf0afbe5cf0d81b15e487c411ae169d3a89" +} diff --git a/backend/.sqlx/query-6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b.json b/backend/.sqlx/query-1cf8597b9d37ec5a924aff8cbc0a05768ed9a679ba908ab16497a9bd55578ba1.json similarity index 75% rename from backend/.sqlx/query-6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b.json rename to backend/.sqlx/query-1cf8597b9d37ec5a924aff8cbc0a05768ed9a679ba908ab16497a9bd55578ba1.json index 6e1b36a97c..b4a9f19f45 100644 --- a/backend/.sqlx/query-6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b.json +++ b/backend/.sqlx/query-1cf8597b9d37ec5a924aff8cbc0a05768ed9a679ba908ab16497a9bd55578ba1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT usr.*, password.super_admin, password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2\n ", + "query": "SELECT usr.*, COALESCE(password.super_admin, false) as \"super_admin!\", password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2\n ", "describe": { "columns": [ { @@ -50,11 +50,16 @@ }, { "ordinal": 9, - "name": "super_admin", + "name": "is_service_account", "type_info": "Bool" }, { "ordinal": 10, + "name": "super_admin!", + "type_info": "Bool" + }, + { + "ordinal": 11, "name": "name", "type_info": "Varchar" } @@ -76,8 +81,9 @@ true, true, false, + null, true ] }, - "hash": "6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b" + "hash": "1cf8597b9d37ec5a924aff8cbc0a05768ed9a679ba908ab16497a9bd55578ba1" } diff --git a/backend/.sqlx/query-1dd73eff0e89b84c0316af2760a136afdd19dc34f9f31c4f9de6b0f74bc386a6.json b/backend/.sqlx/query-1dd73eff0e89b84c0316af2760a136afdd19dc34f9f31c4f9de6b0f74bc386a6.json new file mode 100644 index 0000000000..987acc6d1b --- /dev/null +++ b/backend/.sqlx/query-1dd73eff0e89b84c0316af2760a136afdd19dc34f9f31c4f9de6b0f74bc386a6.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n schemaname || '.' || relname as \"table_name!\",\n pg_total_relation_size(relid) as \"total_size_bytes!\",\n pg_size_pretty(pg_total_relation_size(relid)) as \"total_size_pretty!\"\n FROM pg_catalog.pg_statio_user_tables\n ORDER BY pg_total_relation_size(relid) DESC\n LIMIT 15", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "table_name!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "total_size_bytes!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "total_size_pretty!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "1dd73eff0e89b84c0316af2760a136afdd19dc34f9f31c4f9de6b0f74bc386a6" +} diff --git a/backend/.sqlx/query-250a4e3f1a1f95296f7075bf8780e9c7407e89c8f7636484895e99f5a5e71297.json b/backend/.sqlx/query-250a4e3f1a1f95296f7075bf8780e9c7407e89c8f7636484895e99f5a5e71297.json new file mode 100644 index 0000000000..6fd286898a --- /dev/null +++ b/backend/.sqlx/query-250a4e3f1a1f95296f7075bf8780e9c7407e89c8f7636484895e99f5a5e71297.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr (workspace_id, username, email, is_admin, operator, added_via)\n VALUES ($1, 'alice', 'alice@example.com', false, true, $2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "250a4e3f1a1f95296f7075bf8780e9c7407e89c8f7636484895e99f5a5e71297" +} diff --git a/backend/.sqlx/query-26e62b4509e44a7548957ad4ef217fd46bc03d5dca19344cd3bf7b131fa40ed2.json b/backend/.sqlx/query-26e62b4509e44a7548957ad4ef217fd46bc03d5dca19344cd3bf7b131fa40ed2.json new file mode 100644 index 0000000000..2ab912644a --- /dev/null +++ b/backend/.sqlx/query-26e62b4509e44a7548957ad4ef217fd46bc03d5dca19344cd3bf7b131fa40ed2.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM global_settings WHERE name = 'retention_period_secs'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "26e62b4509e44a7548957ad4ef217fd46bc03d5dca19344cd3bf7b131fa40ed2" +} diff --git a/backend/.sqlx/query-2ba03e555d2e09dbd0e2ae5ddfd9a268a675bdb23615c78904cebe7f1e31f400.json b/backend/.sqlx/query-2ba03e555d2e09dbd0e2ae5ddfd9a268a675bdb23615c78904cebe7f1e31f400.json new file mode 100644 index 0000000000..3f4b2f53ab --- /dev/null +++ b/backend/.sqlx/query-2ba03e555d2e09dbd0e2ae5ddfd9a268a675bdb23615c78904cebe7f1e31f400.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr (workspace_id, username, email, is_admin, operator, added_via)\n VALUES ($1, 'alice', 'alice@example.com', true, false, $2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "2ba03e555d2e09dbd0e2ae5ddfd9a268a675bdb23615c78904cebe7f1e31f400" +} diff --git a/backend/.sqlx/query-2d4ccf3ee19a70cbb5bd034c74703bbb30f217cd3673821e11bae3bf9f925720.json b/backend/.sqlx/query-2d4ccf3ee19a70cbb5bd034c74703bbb30f217cd3673821e11bae3bf9f925720.json new file mode 100644 index 0000000000..007c6fdcb2 --- /dev/null +++ b/backend/.sqlx/query-2d4ccf3ee19a70cbb5bd034c74703bbb30f217cd3673821e11bae3bf9f925720.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n ws.workspace_id as \"workspace_id!\",\n dt.key as \"name!\",\n dt.value->>'table_name' as \"table_name\"\n FROM workspace_settings ws,\n jsonb_each(ws.datatable) dt\n WHERE dt.value->>'resource_type' = 'instance'\n AND dt.value->>'table_name' IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "table_name", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null, + null + ] + }, + "hash": "2d4ccf3ee19a70cbb5bd034c74703bbb30f217cd3673821e11bae3bf9f925720" +} diff --git a/backend/.sqlx/query-2d95191e899d60385b32f36f2e38137e4173a34c54344ee522745640d48b8813.json b/backend/.sqlx/query-2d95191e899d60385b32f36f2e38137e4173a34c54344ee522745640d48b8813.json new file mode 100644 index 0000000000..d31420ada0 --- /dev/null +++ b/backend/.sqlx/query-2d95191e899d60385b32f36f2e38137e4173a34c54344ee522745640d48b8813.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COUNT(*) as \"total!\",\n COUNT(*) FILTER (WHERE state = 'active') as \"active!\",\n COUNT(*) FILTER (WHERE state = 'idle') as \"idle!\"\n FROM pg_stat_activity\n WHERE backend_type = 'client backend'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "total!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "active!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "idle!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "2d95191e899d60385b32f36f2e38137e4173a34c54344ee522745640d48b8813" +} diff --git a/backend/.sqlx/query-30930bfb0513f1a70194a900011b2e890bc4146bb0419210cd76743cacda8bfa.json b/backend/.sqlx/query-30930bfb0513f1a70194a900011b2e890bc4146bb0419210cd76743cacda8bfa.json new file mode 100644 index 0000000000..ca31cbc9ca --- /dev/null +++ b/backend/.sqlx/query-30930bfb0513f1a70194a900011b2e890bc4146bb0419210cd76743cacda8bfa.json @@ -0,0 +1,44 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n table_name as \"table_name!\",\n SUM(live_tuples)::bigint as \"live_tuples!\",\n SUM(dead_tuples)::bigint as \"dead_tuples!\",\n MAX(last_autovacuum) as \"last_autovacuum\",\n MAX(last_autoanalyze) as \"last_autoanalyze\"\n FROM (\n SELECT\n CASE\n WHEN i.inhparent IS NOT NULL THEN schemaname || '.' || p.relname\n ELSE schemaname || '.' || s.relname\n END as table_name,\n COALESCE(n_live_tup, 0) as live_tuples,\n COALESCE(n_dead_tup, 0) as dead_tuples,\n last_autovacuum,\n last_autoanalyze\n FROM pg_stat_user_tables s\n LEFT JOIN pg_class c ON c.relname = s.relname AND c.relnamespace = (\n SELECT oid FROM pg_namespace WHERE nspname = s.schemaname\n )\n LEFT JOIN pg_inherits i ON i.inhrelid = c.oid\n LEFT JOIN pg_class p ON p.oid = i.inhparent\n ) sub\n GROUP BY table_name\n ORDER BY SUM(dead_tuples) DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "table_name!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "live_tuples!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "dead_tuples!", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "last_autovacuum", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "last_autoanalyze", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null, + null, + null + ] + }, + "hash": "30930bfb0513f1a70194a900011b2e890bc4146bb0419210cd76743cacda8bfa" +} diff --git a/backend/.sqlx/query-359cd29f531d263a8cf7205e0869229a610767087f01e0154be8da0620fa114b.json b/backend/.sqlx/query-359cd29f531d263a8cf7205e0869229a610767087f01e0154be8da0620fa114b.json new file mode 100644 index 0000000000..23cf739f90 --- /dev/null +++ b/backend/.sqlx/query-359cd29f531d263a8cf7205e0869229a610767087f01e0154be8da0620fa114b.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH all_audit AS (SELECT username, operation, timestamp FROM audit_partitioned UNION ALL SELECT username, operation, timestamp FROM audit),\n active_users as (SELECT distinct username as email FROM all_audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n active_authors as (SELECT distinct email FROM usr WHERE usr.operator IS false AND email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "authors", + "type_info": "VarcharArray" + }, + { + "ordinal": 1, + "name": "operators", + "type_info": "VarcharArray" + }, + { + "ordinal": 2, + "name": "author_count", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "operator_count", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null, + null + ] + }, + "hash": "359cd29f531d263a8cf7205e0869229a610767087f01e0154be8da0620fa114b" +} diff --git a/backend/.sqlx/query-384f5e9b2ab8e430141e28ea58854cbcfbcf96fd2adbf0513ce942cfe9bceaf0.json b/backend/.sqlx/query-384f5e9b2ab8e430141e28ea58854cbcfbcf96fd2adbf0513ce942cfe9bceaf0.json new file mode 100644 index 0000000000..cb033816d1 --- /dev/null +++ b/backend/.sqlx/query-384f5e9b2ab8e430141e28ea58854cbcfbcf96fd2adbf0513ce942cfe9bceaf0.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_database_size(current_database()) as size_bytes, pg_size_pretty(pg_database_size(current_database())) as size_pretty", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "size_pretty", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "384f5e9b2ab8e430141e28ea58854cbcfbcf96fd2adbf0513ce942cfe9bceaf0" +} diff --git a/backend/.sqlx/query-3bd4f38a1629a69ddda622b6b436198b47c2fe1a507358d49f05031e7beedab6.json b/backend/.sqlx/query-3bd4f38a1629a69ddda622b6b436198b47c2fe1a507358d49f05031e7beedab6.json new file mode 100644 index 0000000000..0c71ee8ad2 --- /dev/null +++ b/backend/.sqlx/query-3bd4f38a1629a69ddda622b6b436198b47c2fe1a507358d49f05031e7beedab6.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT workspace_id,\n auto_invite->'instance_groups_roles' as instance_groups_roles,\n auto_invite->'instance_groups' as instance_groups_json\n FROM workspace_settings\n WHERE auto_invite->'instance_groups' ? $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "instance_groups_roles", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "instance_groups_json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null, + null + ] + }, + "hash": "3bd4f38a1629a69ddda622b6b436198b47c2fe1a507358d49f05031e7beedab6" +} diff --git a/backend/.sqlx/query-446404eda9b9632c9a1384af6bf2f88594825dbaa647290a58bd63df61b531a7.json b/backend/.sqlx/query-446404eda9b9632c9a1384af6bf2f88594825dbaa647290a58bd63df61b531a7.json index e2c5050f1d..9768a13f3d 100644 --- a/backend/.sqlx/query-446404eda9b9632c9a1384af6bf2f88594825dbaa647290a58bd63df61b531a7.json +++ b/backend/.sqlx/query-446404eda9b9632c9a1384af6bf2f88594825dbaa647290a58bd63df61b531a7.json @@ -21,7 +21,7 @@ { "ordinal": 3, "name": "item_path", - "type_info": "Varchar" + "type_info": "Text" }, { "ordinal": 4, diff --git a/backend/.sqlx/query-51c3274a8092d80503a6b97ef3896cc3ba1957042a48ac5f9629ada25b3e78ef.json b/backend/.sqlx/query-51c3274a8092d80503a6b97ef3896cc3ba1957042a48ac5f9629ada25b3e78ef.json index 0f2c7ab318..fa4a6fc50e 100644 --- a/backend/.sqlx/query-51c3274a8092d80503a6b97ef3896cc3ba1957042a48ac5f9629ada25b3e78ef.json +++ b/backend/.sqlx/query-51c3274a8092d80503a6b97ef3896cc3ba1957042a48ac5f9629ada25b3e78ef.json @@ -21,7 +21,7 @@ { "ordinal": 3, "name": "item_path", - "type_info": "Varchar" + "type_info": "Text" }, { "ordinal": 4, diff --git a/backend/.sqlx/query-52d765c87cb8da0ca71fb53156820e383a998a54c95355bb85fe7e762a0d9765.json b/backend/.sqlx/query-52d765c87cb8da0ca71fb53156820e383a998a54c95355bb85fe7e762a0d9765.json new file mode 100644 index 0000000000..16ae512f37 --- /dev/null +++ b/backend/.sqlx/query-52d765c87cb8da0ca71fb53156820e383a998a54c95355bb85fe7e762a0d9765.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT DISTINCT imported_path as \"imported_path!\"\n FROM dependency_map\n WHERE workspace_id = $1\n AND importer_path = $2\n AND imported_path NOT LIKE 'dependencies/%'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "imported_path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "52d765c87cb8da0ca71fb53156820e383a998a54c95355bb85fe7e762a0d9765" +} diff --git a/backend/.sqlx/query-544a02447bb2cbe8354a5c4ae93685848af38a3461257a9734c43cbd7bd905cb.json b/backend/.sqlx/query-544a02447bb2cbe8354a5c4ae93685848af38a3461257a9734c43cbd7bd905cb.json new file mode 100644 index 0000000000..2086242e9c --- /dev/null +++ b/backend/.sqlx/query-544a02447bb2cbe8354a5c4ae93685848af38a3461257a9734c43cbd7bd905cb.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, is_service_account, disabled FROM usr WHERE username = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_service_account", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "disabled", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "544a02447bb2cbe8354a5c4ae93685848af38a3461257a9734c43cbd7bd905cb" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-5d160ba4958583f1ad42de846c544d8d8e81e1b54925a0c5f2cedc1817d99a1b.json b/backend/.sqlx/query-5d160ba4958583f1ad42de846c544d8d8e81e1b54925a0c5f2cedc1817d99a1b.json new file mode 100644 index 0000000000..8fcffe364a --- /dev/null +++ b/backend/.sqlx/query-5d160ba4958583f1ad42de846c544d8d8e81e1b54925a0c5f2cedc1817d99a1b.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT igroup FROM email_to_igroup WHERE email = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "igroup", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5d160ba4958583f1ad42de846c544d8d8e81e1b54925a0c5f2cedc1817d99a1b" +} diff --git a/backend/.sqlx/query-5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c.json b/backend/.sqlx/query-5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c.json index 09775dcc3a..79625b6baf 100644 --- a/backend/.sqlx/query-5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c.json +++ b/backend/.sqlx/query-5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c.json @@ -47,6 +47,11 @@ "ordinal": 8, "name": "added_via", "type_info": "Jsonb" + }, + { + "ordinal": 9, + "name": "is_service_account", + "type_info": "Bool" } ], "parameters": { @@ -63,7 +68,8 @@ false, false, true, - true + true, + false ] }, "hash": "5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c" diff --git a/backend/.sqlx/query-60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b.json b/backend/.sqlx/query-60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b.json index 3a635ab004..ed09f2833f 100644 --- a/backend/.sqlx/query-60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b.json +++ b/backend/.sqlx/query-60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b.json @@ -47,6 +47,11 @@ "ordinal": 8, "name": "added_via", "type_info": "Jsonb" + }, + { + "ordinal": 9, + "name": "is_service_account", + "type_info": "Bool" } ], "parameters": { @@ -64,7 +69,8 @@ false, false, true, - true + true, + false ] }, "hash": "60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b" diff --git a/backend/.sqlx/query-62e8e443cf063fcb30799d9c8971c00d761d54811936deb87a0315ca9cdc9769.json b/backend/.sqlx/query-62e8e443cf063fcb30799d9c8971c00d761d54811936deb87a0315ca9cdc9769.json new file mode 100644 index 0000000000..f4a2546900 --- /dev/null +++ b/backend/.sqlx/query-62e8e443cf063fcb30799d9c8971c00d761d54811936deb87a0315ca9cdc9769.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT is_admin, operator, added_via FROM usr WHERE workspace_id = 'ws-multi-group' AND email = 'alice@example.com'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_admin", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "operator", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "added_via", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "62e8e443cf063fcb30799d9c8971c00d761d54811936deb87a0315ca9cdc9769" +} diff --git a/backend/.sqlx/query-7e01ef5799168c0fc2779d42ce352827e2fda6711c0a1b104ca6435ddb14b47d.json b/backend/.sqlx/query-66e2f8468ba64f22b7a7caa18639d7c833ac2ec573bd89d878b5c8b1afc74d3a.json similarity index 54% rename from backend/.sqlx/query-7e01ef5799168c0fc2779d42ce352827e2fda6711c0a1b104ca6435ddb14b47d.json rename to backend/.sqlx/query-66e2f8468ba64f22b7a7caa18639d7c833ac2ec573bd89d878b5c8b1afc74d3a.json index 7c8a5bc156..15681aab51 100644 --- a/backend/.sqlx/query-7e01ef5799168c0fc2779d42ce352827e2fda6711c0a1b104ca6435ddb14b47d.json +++ b/backend/.sqlx/query-66e2f8468ba64f22b7a7caa18639d7c833ac2ec573bd89d878b5c8b1afc74d3a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n workspace_id,\n auto_invite->'instance_groups_roles' as instance_groups_roles\n FROM workspace_settings\n WHERE\n auto_invite->'instance_groups' IS NOT NULL\n AND auto_invite->'instance_groups' ? $1\n ", + "query": "\n SELECT\n workspace_id,\n auto_invite->'instance_groups_roles' as instance_groups_roles,\n auto_invite->'instance_groups' as instance_groups_json\n FROM workspace_settings\n WHERE\n auto_invite->'instance_groups' IS NOT NULL\n AND auto_invite->'instance_groups' ? $1\n ", "describe": { "columns": [ { @@ -12,6 +12,11 @@ "ordinal": 1, "name": "instance_groups_roles", "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "instance_groups_json", + "type_info": "Jsonb" } ], "parameters": { @@ -21,8 +26,9 @@ }, "nullable": [ false, + null, null ] }, - "hash": "7e01ef5799168c0fc2779d42ce352827e2fda6711c0a1b104ca6435ddb14b47d" + "hash": "66e2f8468ba64f22b7a7caa18639d7c833ac2ec573bd89d878b5c8b1afc74d3a" } diff --git a/backend/.sqlx/query-68c19cb0e18b94870bbe81f9aab92ba37da67cd2a56834c9d1378eab7551284d.json b/backend/.sqlx/query-68c19cb0e18b94870bbe81f9aab92ba37da67cd2a56834c9d1378eab7551284d.json new file mode 100644 index 0000000000..de69c0fc8c --- /dev/null +++ b/backend/.sqlx/query-68c19cb0e18b94870bbe81f9aab92ba37da67cd2a56834c9d1378eab7551284d.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE kafka_trigger\n SET\n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n filters = $4,\n filter_logic = $5,\n auto_offset_reset = $6,\n auto_commit = $7,\n script_path = $8,\n path = $9,\n is_flow = $10,\n edited_by = $11,\n permissioned_as = $12,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $15,\n error_handler_args = $16,\n retry = $17\n WHERE\n workspace_id = $13 AND path = $14\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "VarcharArray", + "JsonbArray", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "68c19cb0e18b94870bbe81f9aab92ba37da67cd2a56834c9d1378eab7551284d" +} diff --git a/backend/.sqlx/query-68d1370fa02f4fe585684a91e898c4aed45e6b8f409bb33c2681f92265922040.json b/backend/.sqlx/query-68d1370fa02f4fe585684a91e898c4aed45e6b8f409bb33c2681f92265922040.json new file mode 100644 index 0000000000..cd1d6810cd --- /dev/null +++ b/backend/.sqlx/query-68d1370fa02f4fe585684a91e898c4aed45e6b8f409bb33c2681f92265922040.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND (username = $2 OR email = $3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "68d1370fa02f4fe585684a91e898c4aed45e6b8f409bb33c2681f92265922040" +} diff --git a/backend/.sqlx/query-942c0abb55c910862fd45d3fa56a4eb6729f1a658101bda2d0b0fca96b3cfee5.json b/backend/.sqlx/query-6948eb5aabf82f2f4a08dd4410eb472080ecab3ed652912397245e5216ae0389.json similarity index 60% rename from backend/.sqlx/query-942c0abb55c910862fd45d3fa56a4eb6729f1a658101bda2d0b0fca96b3cfee5.json rename to backend/.sqlx/query-6948eb5aabf82f2f4a08dd4410eb472080ecab3ed652912397245e5216ae0389.json index e3ca43fbd0..a6f1f5f7cc 100644 --- a/backend/.sqlx/query-942c0abb55c910862fd45d3fa56a4eb6729f1a658101bda2d0b0fca96b3cfee5.json +++ b/backend/.sqlx/query-6948eb5aabf82f2f4a08dd4410eb472080ecab3ed652912397245e5216ae0389.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO websocket_trigger (\n workspace_id,\n path,\n url,\n script_path,\n is_flow,\n mode,\n filters,\n initial_messages,\n url_runnable_args,\n edited_by,\n can_return_message,\n can_return_error_result,\n permissioned_as,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), $14, $15, $16\n )\n ", + "query": "\n INSERT INTO websocket_trigger (\n workspace_id,\n path,\n url,\n script_path,\n is_flow,\n mode,\n filters,\n filter_logic,\n initial_messages,\n url_runnable_args,\n edited_by,\n can_return_message,\n can_return_error_result,\n permissioned_as,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17\n )\n ", "describe": { "columns": [], "parameters": { @@ -23,6 +23,7 @@ } }, "JsonbArray", + "Varchar", "JsonbArray", "Jsonb", "Varchar", @@ -36,5 +37,5 @@ }, "nullable": [] }, - "hash": "942c0abb55c910862fd45d3fa56a4eb6729f1a658101bda2d0b0fca96b3cfee5" + "hash": "6948eb5aabf82f2f4a08dd4410eb472080ecab3ed652912397245e5216ae0389" } diff --git a/backend/.sqlx/query-a0a545fda5f3ebea0113d5daaf13358c964d9fb0f41bf2a1c834305b4d2398f2.json b/backend/.sqlx/query-6a8f4ed9946bb2a3c5e90695c90b70aa2e83fcb5aa0c953febdd9bac2d95bbec.json similarity index 57% rename from backend/.sqlx/query-a0a545fda5f3ebea0113d5daaf13358c964d9fb0f41bf2a1c834305b4d2398f2.json rename to backend/.sqlx/query-6a8f4ed9946bb2a3c5e90695c90b70aa2e83fcb5aa0c953febdd9bac2d95bbec.json index 99d8d2a181..c6d3374639 100644 --- a/backend/.sqlx/query-a0a545fda5f3ebea0113d5daaf13358c964d9fb0f41bf2a1c834305b4d2398f2.json +++ b/backend/.sqlx/query-6a8f4ed9946bb2a3c5e90695c90b70aa2e83fcb5aa0c953febdd9bac2d95bbec.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n filters,\n auto_offset_reset,\n auto_commit,\n script_path,\n is_flow,\n mode,\n edited_by,\n permissioned_as,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), $14, $15, $16\n )\n ", + "query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n filters,\n filter_logic,\n auto_offset_reset,\n auto_commit,\n script_path,\n is_flow,\n mode,\n edited_by,\n permissioned_as,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17\n )\n ", "describe": { "columns": [], "parameters": { @@ -12,6 +12,7 @@ "VarcharArray", "JsonbArray", "Varchar", + "Varchar", "Bool", "Varchar", "Bool", @@ -36,5 +37,5 @@ }, "nullable": [] }, - "hash": "a0a545fda5f3ebea0113d5daaf13358c964d9fb0f41bf2a1c834305b4d2398f2" + "hash": "6a8f4ed9946bb2a3c5e90695c90b70aa2e83fcb5aa0c953febdd9bac2d95bbec" } diff --git a/backend/.sqlx/query-6d070f476538aa6fcd6227fe5312561a7d098f2af5287e1e6c339e15080378be.json b/backend/.sqlx/query-6d070f476538aa6fcd6227fe5312561a7d098f2af5287e1e6c339e15080378be.json new file mode 100644 index 0000000000..6d6acec840 --- /dev/null +++ b/backend/.sqlx/query-6d070f476538aa6fcd6227fe5312561a7d098f2af5287e1e6c339e15080378be.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext($1 || '/' || $2))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "6d070f476538aa6fcd6227fe5312561a7d098f2af5287e1e6c339e15080378be" +} diff --git a/backend/.sqlx/query-6f941e4454f736b32eaef80cdfb9582d6e75af3dc159e7c5f12497d3957f1eef.json b/backend/.sqlx/query-6f941e4454f736b32eaef80cdfb9582d6e75af3dc159e7c5f12497d3957f1eef.json new file mode 100644 index 0000000000..8446e14aa4 --- /dev/null +++ b/backend/.sqlx/query-6f941e4454f736b32eaef80cdfb9582d6e75af3dc159e7c5f12497d3957f1eef.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr (workspace_id, username, email, is_admin, operator)\n VALUES ('ws-multi-group', 'alice', 'alice@example.com', true, false)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "6f941e4454f736b32eaef80cdfb9582d6e75af3dc159e7c5f12497d3957f1eef" +} diff --git a/backend/.sqlx/query-7c5db0b3bd1dd1f766e1841ca620871a468033e05b6e0188ea4775b63fc66e84.json b/backend/.sqlx/query-7c5db0b3bd1dd1f766e1841ca620871a468033e05b6e0188ea4775b63fc66e84.json new file mode 100644 index 0000000000..304027071a --- /dev/null +++ b/backend/.sqlx/query-7c5db0b3bd1dd1f766e1841ca620871a468033e05b6e0188ea4775b63fc66e84.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n c.relname as \"table_name!\",\n pg_total_relation_size(c.oid) as \"size_bytes!\",\n pg_size_pretty(pg_total_relation_size(c.oid)) as \"size_pretty!\",\n COALESCE(c.reltuples, 0) as \"estimated_rows!\"\n FROM pg_class c\n JOIN pg_namespace n ON n.oid = c.relnamespace\n WHERE n.nspname = 'public' AND c.relname = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "table_name!", + "type_info": "Name" + }, + { + "ordinal": 1, + "name": "size_bytes!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "size_pretty!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "estimated_rows!", + "type_info": "Float4" + } + ], + "parameters": { + "Left": [ + "NameArray" + ] + }, + "nullable": [ + false, + null, + null, + null + ] + }, + "hash": "7c5db0b3bd1dd1f766e1841ca620871a468033e05b6e0188ea4775b63fc66e84" +} diff --git a/backend/.sqlx/query-88a467f3c943b134a81ac69c3c6686d1ce1ff2f5aafc15ff1b63cfa86c09c4f0.json b/backend/.sqlx/query-88a467f3c943b134a81ac69c3c6686d1ce1ff2f5aafc15ff1b63cfa86c09c4f0.json new file mode 100644 index 0000000000..6ee3c89e40 --- /dev/null +++ b/backend/.sqlx/query-88a467f3c943b134a81ac69c3c6686d1ce1ff2f5aafc15ff1b63cfa86c09c4f0.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT is_admin, operator FROM usr WHERE workspace_id = 'ws-multi-group' AND email = 'alice@example.com'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_admin", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "operator", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false + ] + }, + "hash": "88a467f3c943b134a81ac69c3c6686d1ce1ff2f5aafc15ff1b63cfa86c09c4f0" +} diff --git a/backend/.sqlx/query-8b25c4252da77cd2fe1b3916b518251dbb3c6d4c095efa015823f0324ab27d7f.json b/backend/.sqlx/query-8b25c4252da77cd2fe1b3916b518251dbb3c6d4c095efa015823f0324ab27d7f.json index 4692b430ec..9995bb1b51 100644 --- a/backend/.sqlx/query-8b25c4252da77cd2fe1b3916b518251dbb3c6d4c095efa015823f0324ab27d7f.json +++ b/backend/.sqlx/query-8b25c4252da77cd2fe1b3916b518251dbb3c6d4c095efa015823f0324ab27d7f.json @@ -13,7 +13,7 @@ "Left": [ "Varchar", "Varchar", - "Varchar", + "Text", "Jsonb", "Varchar" ] diff --git a/backend/.sqlx/query-92fb6afe3b7041b2954340094c08e702fc1577d3fa4ff1ff2f1e089971ff5e32.json b/backend/.sqlx/query-92fb6afe3b7041b2954340094c08e702fc1577d3fa4ff1ff2f1e089971ff5e32.json index 191630bd35..49fd50d7e6 100644 --- a/backend/.sqlx/query-92fb6afe3b7041b2954340094c08e702fc1577d3fa4ff1ff2f1e089971ff5e32.json +++ b/backend/.sqlx/query-92fb6afe3b7041b2954340094c08e702fc1577d3fa4ff1ff2f1e089971ff5e32.json @@ -21,7 +21,7 @@ { "ordinal": 3, "name": "item_path", - "type_info": "Varchar" + "type_info": "Text" }, { "ordinal": 4, diff --git a/backend/.sqlx/query-726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de.json b/backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json similarity index 52% rename from backend/.sqlx/query-726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de.json rename to backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json index 4baf932023..e751b8fc7c 100644 --- a/backend/.sqlx/query-726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de.json +++ b/backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC", + "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC", "describe": { "columns": [ { @@ -12,6 +12,11 @@ "ordinal": 1, "name": "deployment_msg", "type_info": "Text" + }, + { + "ordinal": 2, + "name": "created_at", + "type_info": "Timestamptz" } ], "parameters": { @@ -22,8 +27,9 @@ }, "nullable": [ false, - true + true, + false ] }, - "hash": "726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de" + "hash": "9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c" } diff --git a/backend/.sqlx/query-a37c2c4d5656d4b44433de84c454046f7586e36b7bd6a4679d70c359d4aacfcf.json b/backend/.sqlx/query-a37c2c4d5656d4b44433de84c454046f7586e36b7bd6a4679d70c359d4aacfcf.json new file mode 100644 index 0000000000..5661c59faf --- /dev/null +++ b/backend/.sqlx/query-a37c2c4d5656d4b44433de84c454046f7586e36b7bd6a4679d70c359d4aacfcf.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM usr WHERE is_service_account = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "a37c2c4d5656d4b44433de84c454046f7586e36b7bd6a4679d70c359d4aacfcf" +} diff --git a/backend/.sqlx/query-a37cfc632dd37cf37c06743239b5ebc784e5da5ee25d47af187a75220d8fded7.json b/backend/.sqlx/query-a37cfc632dd37cf37c06743239b5ebc784e5da5ee25d47af187a75220d8fded7.json deleted file mode 100644 index c993120dae..0000000000 --- a/backend/.sqlx/query-a37cfc632dd37cf37c06743239b5ebc784e5da5ee25d47af187a75220d8fded7.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE kafka_trigger\n SET\n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n filters = $4,\n auto_offset_reset = $5,\n auto_commit = $6,\n script_path = $7,\n path = $8,\n is_flow = $9,\n edited_by = $10,\n permissioned_as = $11,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE\n workspace_id = $12 AND path = $13\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "VarcharArray", - "JsonbArray", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "a37cfc632dd37cf37c06743239b5ebc784e5da5ee25d47af187a75220d8fded7" -} diff --git a/backend/.sqlx/query-a860dd9722f608184c4b1ef5e609b20cd61f9967a2012fc1c8fe352ee7596358.json b/backend/.sqlx/query-a860dd9722f608184c4b1ef5e609b20cd61f9967a2012fc1c8fe352ee7596358.json new file mode 100644 index 0000000000..539c029265 --- /dev/null +++ b/backend/.sqlx/query-a860dd9722f608184c4b1ef5e609b20cd61f9967a2012fc1c8fe352ee7596358.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('7 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = EXCLUDED.lockfile, expiration = EXCLUDED.expiration", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a860dd9722f608184c4b1ef5e609b20cd61f9967a2012fc1c8fe352ee7596358" +} diff --git a/backend/.sqlx/query-a9c3461ca3053f699c957f61780d1e889ad53dc5bf1669c24c0666c290656c00.json b/backend/.sqlx/query-a9c3461ca3053f699c957f61780d1e889ad53dc5bf1669c24c0666c290656c00.json new file mode 100644 index 0000000000..cb6ed918c4 --- /dev/null +++ b/backend/.sqlx/query-a9c3461ca3053f699c957f61780d1e889ad53dc5bf1669c24c0666c290656c00.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT AVG(pg_column_size(result))::bigint as \"avg_size\"\n FROM (\n SELECT result FROM v2_job_completed\n WHERE completed_at > now() - interval '30 days'\n AND result IS NOT NULL\n ORDER BY completed_at DESC\n LIMIT $1\n ) sub", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "avg_size", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a9c3461ca3053f699c957f61780d1e889ad53dc5bf1669c24c0666c290656c00" +} diff --git a/backend/.sqlx/query-add01e9e31d64e88b84c9505fe3de553031e581b1bb173413a9a3e3eb0817b43.json b/backend/.sqlx/query-add01e9e31d64e88b84c9505fe3de553031e581b1bb173413a9a3e3eb0817b43.json new file mode 100644 index 0000000000..2c00759a63 --- /dev/null +++ b/backend/.sqlx/query-add01e9e31d64e88b84c9505fe3de553031e581b1bb173413a9a3e3eb0817b43.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "add01e9e31d64e88b84c9505fe3de553031e581b1bb173413a9a3e3eb0817b43" +} diff --git a/backend/.sqlx/query-b38bd869477a729279cac3ccd4825191fb49e17e5f7e7297c0c819f52b486f49.json b/backend/.sqlx/query-b38bd869477a729279cac3ccd4825191fb49e17e5f7e7297c0c819f52b486f49.json new file mode 100644 index 0000000000..955fee77ad --- /dev/null +++ b/backend/.sqlx/query-b38bd869477a729279cac3ccd4825191fb49e17e5f7e7297c0c819f52b486f49.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE usr SET is_admin = $1, operator = $2, added_via = $3 WHERE workspace_id = $4 AND email = $5 AND added_via->>'source' = 'instance_group'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Bool", + "Jsonb", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b38bd869477a729279cac3ccd4825191fb49e17e5f7e7297c0c819f52b486f49" +} diff --git a/backend/.sqlx/query-b760be4a0a80853073a061f7c9ebc2d411294d57b07d54d15d178db3c6ee2a30.json b/backend/.sqlx/query-b760be4a0a80853073a061f7c9ebc2d411294d57b07d54d15d178db3c6ee2a30.json new file mode 100644 index 0000000000..a01296f654 --- /dev/null +++ b/backend/.sqlx/query-b760be4a0a80853073a061f7c9ebc2d411294d57b07d54d15d178db3c6ee2a30.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT MIN(completed_at) as oldest, COUNT(*) as total FROM v2_job_completed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "oldest", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "total", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "b760be4a0a80853073a061f7c9ebc2d411294d57b07d54d15d178db3c6ee2a30" +} diff --git a/backend/.sqlx/query-cf2a6ad6471a40b6298775cda9300aeecdd75503bed59d80cd62091d1642d1ec.json b/backend/.sqlx/query-c73e98e5a937f44724a96ee1b74d31fa71a7be3b8ba3dec9f59f54a6c4030462.json similarity index 51% rename from backend/.sqlx/query-cf2a6ad6471a40b6298775cda9300aeecdd75503bed59d80cd62091d1642d1ec.json rename to backend/.sqlx/query-c73e98e5a937f44724a96ee1b74d31fa71a7be3b8ba3dec9f59f54a6c4030462.json index 4e17af3ce2..06b2c50058 100644 --- a/backend/.sqlx/query-cf2a6ad6471a40b6298775cda9300aeecdd75503bed59d80cd62091d1642d1ec.json +++ b/backend/.sqlx/query-c73e98e5a937f44724a96ee1b74d31fa71a7be3b8ba3dec9f59f54a6c4030462.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC LIMIT 1", + "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC LIMIT 1", "describe": { "columns": [ { @@ -12,6 +12,11 @@ "ordinal": 1, "name": "deployment_msg", "type_info": "Text" + }, + { + "ordinal": 2, + "name": "created_at", + "type_info": "Timestamptz" } ], "parameters": { @@ -22,8 +27,9 @@ }, "nullable": [ false, - true + true, + false ] }, - "hash": "cf2a6ad6471a40b6298775cda9300aeecdd75503bed59d80cd62091d1642d1ec" + "hash": "c73e98e5a937f44724a96ee1b74d31fa71a7be3b8ba3dec9f59f54a6c4030462" } diff --git a/backend/.sqlx/query-c7aed7fe3b6774477d403bc3e7fcbce7cdbdd1feb553718cbde60bb8ccff4733.json b/backend/.sqlx/query-c7aed7fe3b6774477d403bc3e7fcbce7cdbdd1feb553718cbde60bb8ccff4733.json deleted file mode 100644 index 772b86a11c..0000000000 --- a/backend/.sqlx/query-c7aed7fe3b6774477d403bc3e7fcbce7cdbdd1feb553718cbde60bb8ccff4733.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE\n websocket_trigger\n SET\n url = $1,\n script_path = $2,\n path = $3,\n is_flow = $4,\n filters = $5,\n initial_messages = $6,\n url_runnable_args = $7,\n edited_by = $8,\n permissioned_as = $9,\n can_return_message = $10,\n can_return_error_result = $11,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE\n workspace_id = $12 AND path = $13\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Bool", - "JsonbArray", - "JsonbArray", - "Jsonb", - "Varchar", - "Varchar", - "Bool", - "Bool", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "c7aed7fe3b6774477d403bc3e7fcbce7cdbdd1feb553718cbde60bb8ccff4733" -} diff --git a/backend/.sqlx/query-cb3862634f18160207ee2621ddfca43f00456a27fda32583846497116f92f96c.json b/backend/.sqlx/query-cb3862634f18160207ee2621ddfca43f00456a27fda32583846497116f92f96c.json deleted file mode 100644 index 8ccf623719..0000000000 --- a/backend/.sqlx/query-cb3862634f18160207ee2621ddfca43f00456a27fda32583846497116f92f96c.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH active_users as (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n active_authors as (SELECT distinct email FROM usr WHERE usr.operator IS false AND email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "authors", - "type_info": "VarcharArray" - }, - { - "ordinal": 1, - "name": "operators", - "type_info": "VarcharArray" - }, - { - "ordinal": 2, - "name": "author_count", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "operator_count", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - null, - null, - null - ] - }, - "hash": "cb3862634f18160207ee2621ddfca43f00456a27fda32583846497116f92f96c" -} diff --git a/backend/.sqlx/query-d43a4ff78e48580815fb912c98639a08d45596a9f11a2dcf5b1e0d135844ecda.json b/backend/.sqlx/query-d43a4ff78e48580815fb912c98639a08d45596a9f11a2dcf5b1e0d135844ecda.json new file mode 100644 index 0000000000..1cfd1e268c --- /dev/null +++ b/backend/.sqlx/query-d43a4ff78e48580815fb912c98639a08d45596a9f11a2dcf5b1e0d135844ecda.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM global_settings WHERE name = 'plain_emails_telemetry'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "d43a4ff78e48580815fb912c98639a08d45596a9f11a2dcf5b1e0d135844ecda" +} diff --git a/backend/.sqlx/query-dbc5924bca3aa0b32e296b73f8a967bed68332caf526216597f10ffa5fa951c7.json b/backend/.sqlx/query-dbc5924bca3aa0b32e296b73f8a967bed68332caf526216597f10ffa5fa951c7.json new file mode 100644 index 0000000000..33eb1fbf7e --- /dev/null +++ b/backend/.sqlx/query-dbc5924bca3aa0b32e296b73f8a967bed68332caf526216597f10ffa5fa951c7.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n c.id as \"id!\",\n c.workspace_id as \"workspace_id!\",\n j.runnable_path as \"runnable_path\",\n pg_column_size(c.result) as \"result_size_bytes!\",\n c.completed_at as \"completed_at!\"\n FROM (\n SELECT id, workspace_id, result, completed_at\n FROM v2_job_completed\n WHERE completed_at > now() - interval '30 days'\n AND result IS NOT NULL\n ORDER BY completed_at DESC\n LIMIT $1\n ) c\n LEFT JOIN v2_job j ON j.id = c.id\n WHERE pg_column_size(c.result) > 1024\n ORDER BY pg_column_size(c.result) DESC\n LIMIT 10", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "result_size_bytes!", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "completed_at!", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + null, + false + ] + }, + "hash": "dbc5924bca3aa0b32e296b73f8a967bed68332caf526216597f10ffa5fa951c7" +} diff --git a/backend/.sqlx/query-e3d4f89ce36337af15d237b543eaca47771b480ff194884f9c947dcaf71d6cf9.json b/backend/.sqlx/query-e3d4f89ce36337af15d237b543eaca47771b480ff194884f9c947dcaf71d6cf9.json new file mode 100644 index 0000000000..8ff6f2e89c --- /dev/null +++ b/backend/.sqlx/query-e3d4f89ce36337af15d237b543eaca47771b480ff194884f9c947dcaf71d6cf9.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n websocket_trigger\n SET\n url = $1,\n script_path = $2,\n path = $3,\n is_flow = $4,\n filters = $5,\n filter_logic = $6,\n initial_messages = $7,\n url_runnable_args = $8,\n edited_by = $9,\n permissioned_as = $10,\n can_return_message = $11,\n can_return_error_result = $12,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $15,\n error_handler_args = $16,\n retry = $17\n WHERE\n workspace_id = $13 AND path = $14\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Bool", + "JsonbArray", + "Varchar", + "JsonbArray", + "Jsonb", + "Varchar", + "Varchar", + "Bool", + "Bool", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "e3d4f89ce36337af15d237b543eaca47771b480ff194884f9c947dcaf71d6cf9" +} diff --git a/backend/.sqlx/query-e58ef252b0d2b81e9cd76f394a396abefd791906ada29dd5a7a9148157635ca5.json b/backend/.sqlx/query-e58ef252b0d2b81e9cd76f394a396abefd791906ada29dd5a7a9148157635ca5.json new file mode 100644 index 0000000000..855c628c1f --- /dev/null +++ b/backend/.sqlx/query-e58ef252b0d2b81e9cd76f394a396abefd791906ada29dd5a7a9148157635ca5.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT auto_invite->'instance_groups_roles' as instance_groups_roles,\n auto_invite->'instance_groups' as instance_groups_json\n FROM workspace_settings WHERE workspace_id = 'ws-multi-group'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "instance_groups_roles", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "instance_groups_json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "e58ef252b0d2b81e9cd76f394a396abefd791906ada29dd5a7a9148157635ca5" +} diff --git a/backend/.sqlx/query-e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2.json b/backend/.sqlx/query-e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2.json index cdeb30f672..7be961c050 100644 --- a/backend/.sqlx/query-e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2.json +++ b/backend/.sqlx/query-e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2.json @@ -47,6 +47,11 @@ "ordinal": 8, "name": "added_via", "type_info": "Jsonb" + }, + { + "ordinal": 9, + "name": "is_service_account", + "type_info": "Bool" } ], "parameters": { @@ -63,7 +68,8 @@ false, false, true, - true + true, + false ] }, "hash": "e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2" diff --git a/backend/.sqlx/query-f4ad2cf2438c2ae31e388517d09a2c1a2f63ab88cdbc79ffad96c6f9ffb5764b.json b/backend/.sqlx/query-f4ad2cf2438c2ae31e388517d09a2c1a2f63ab88cdbc79ffad96c6f9ffb5764b.json new file mode 100644 index 0000000000..24d9cd9517 --- /dev/null +++ b/backend/.sqlx/query-f4ad2cf2438c2ae31e388517d09a2c1a2f63ab88cdbc79ffad96c6f9ffb5764b.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO token\n (token_hash, token_prefix, token, email, label, expiration, super_admin, owner)\n VALUES ($1, $2, $3, $4, $5, $6, false, $7)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Timestamptz", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "f4ad2cf2438c2ae31e388517d09a2c1a2f63ab88cdbc79ffad96c6f9ffb5764b" +} diff --git a/backend/.sqlx/query-f8654d5f50a80d862edbf57355502a9bd039d16f7dfb11e22d16ff9090456853.json b/backend/.sqlx/query-f8654d5f50a80d862edbf57355502a9bd039d16f7dfb11e22d16ff9090456853.json new file mode 100644 index 0000000000..117a8bdc1b --- /dev/null +++ b/backend/.sqlx/query-f8654d5f50a80d862edbf57355502a9bd039d16f7dfb11e22d16ff9090456853.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr\n (workspace_id, email, username, is_admin, operator, is_service_account)\n VALUES ($1, $2, $3, false, true, true)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "f8654d5f50a80d862edbf57355502a9bd039d16f7dfb11e22d16ff9090456853" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 684fef7714..10f4348421 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -39,7 +39,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -65,7 +65,7 @@ checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ "cfg-if", "cipher 0.3.0", - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", ] @@ -77,7 +77,7 @@ checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" dependencies = [ "cfg-if", "cipher 0.4.4", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -249,7 +249,7 @@ checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" dependencies = [ "base64ct", "blake2", - "cpufeatures", + "cpufeatures 0.2.17", "password-hash", ] @@ -602,7 +602,7 @@ dependencies = [ "futures-core", "libc", "portable-atomic", - "rustc-hash 2.1.1", + "rustc-hash 2.1.2", "tokio", "tokio-stream", "xattr", @@ -870,9 +870,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.39.0" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a" +checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" dependencies = [ "cc", "cmake", @@ -1113,7 +1113,7 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", "http 1.4.0", "percent-encoding", @@ -1202,7 +1202,7 @@ dependencies = [ "http 1.4.0", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-rustls 0.24.2", "hyper-rustls 0.27.7", "hyper-util", @@ -1363,32 +1363,23 @@ checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", "axum-core 0.4.5", - "axum-macros", "bytes", "futures-util", "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.8.1", - "hyper-util", "itoa", "matchit 0.7.3", "memchr", "mime", - "multer", "percent-encoding", "pin-project-lite", "rustversion", "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", "sync_wrapper", - "tokio", "tower 0.5.3", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -1398,18 +1389,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" dependencies = [ "axum-core 0.5.6", + "axum-macros", "bytes", "form_urlencoded", "futures-util", "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-util", "itoa", "matchit 0.8.4", "memchr", "mime", + "multer", "percent-encoding", "pin-project-lite", "rustversion", @@ -1443,7 +1436,6 @@ dependencies = [ "sync_wrapper", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -1467,9 +1459,9 @@ dependencies = [ [[package]] name = "axum-macros" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" +checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" dependencies = [ "proc-macro2", "quote", @@ -1629,7 +1621,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.1", + "rustc-hash 2.1.2", "shlex", "syn 2.0.117", ] @@ -1649,7 +1641,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.1", + "rustc-hash 2.1.2", "shlex", "syn 2.0.117", ] @@ -1731,16 +1723,16 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq 0.4.2", - "cpufeatures", + "cpufeatures 0.3.0", ] [[package]] @@ -1768,6 +1760,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-modes" version = "0.8.1" @@ -1807,7 +1808,7 @@ dependencies = [ "hex", "http 1.4.0", "http-body-util", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-named-pipe", "hyper-util", "hyperlocal", @@ -2188,9 +2189,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.57" +version = "1.2.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" dependencies = [ "find-msvc-tools", "jobserver", @@ -2237,6 +2238,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.0", +] + [[package]] name = "chrono" version = "0.4.44" @@ -2286,7 +2298,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] @@ -2352,13 +2364,19 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0758edba32d61d1fd9f4d69491b47604b91ee2f7e6b33de7e54ca4ebe55dc3" + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -2456,6 +2474,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -2539,16 +2563,6 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147be55d677052dabc6b22252d5dd0fd4c29c8c27aa4f2fbef0f94aa003b406f" -[[package]] -name = "cookie" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7efb37c3e1ccb1ff97164ad95ac1606e8ccd35b3fa0a7d99a304c7f4a428cc24" -dependencies = [ - "time", - "version_check", -] - [[package]] name = "cookie" version = "0.18.1" @@ -2606,6 +2620,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -2741,6 +2764,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.3.1" @@ -2771,6 +2803,15 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "ctutils" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1005a6d4446f5120ef475ad3d2af2b30c49c2c9c6904258e3bb30219bebed5e4" +dependencies = [ + "cmov", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -2778,7 +2819,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto 0.2.9", @@ -3865,7 +3906,7 @@ dependencies = [ "aes-kw", "base64 0.21.7", "cbc", - "const-oid", + "const-oid 0.9.6", "ctr", "curve25519-dalek", "deno_core", @@ -3940,7 +3981,7 @@ dependencies = [ "hickory-resolver", "http 1.4.0", "http-body-util", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-rustls 0.27.7", "hyper-util", "ipnet", @@ -4028,7 +4069,7 @@ dependencies = [ "http 1.4.0", "httparse", "hyper 0.14.32", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-util", "itertools 0.10.5", "memmem", @@ -4194,7 +4235,7 @@ dependencies = [ "brotli 6.0.0", "bytes", "cbc", - "const-oid", + "const-oid 0.9.6", "ctr", "data-encoding", "deno_core", @@ -4221,7 +4262,7 @@ dependencies = [ "hkdf", "http 1.4.0", "http-body-util", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-util", "idna", "indexmap 2.12.0", @@ -4492,7 +4533,7 @@ dependencies = [ "http 1.4.0", "http-body-util", "hyper 0.14.32", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-util", "libc", "log", @@ -4546,15 +4587,15 @@ dependencies = [ "deno_error", "deno_tls", "http-body-util", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-rustls 0.27.7", "hyper-util", "log", "once_cell", "opentelemetry 0.27.1", - "opentelemetry-http", - "opentelemetry-otlp", - "opentelemetry-semantic-conventions", + "opentelemetry-http 0.27.0", + "opentelemetry-otlp 0.27.0", + "opentelemetry-semantic-conventions 0.27.0", "opentelemetry_sdk 0.27.1", "pin-project", "serde", @@ -4676,7 +4717,7 @@ dependencies = [ "h2 0.4.13", "http 1.4.0", "http-body-util", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-util", "once_cell", "rustls-tokio-stream", @@ -4781,7 +4822,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der_derive", "pem-rfc7468", "zeroize", @@ -4967,11 +5008,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid", - "crypto-common", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4850db49bf08e663084f7fb5c87d202ef91a3907271aff24a94eb97ff039153c" +dependencies = [ + "block-buffer 0.12.0", + "const-oid 0.10.2", + "crypto-common 0.2.1", + "ctutils", +] + [[package]] name = "dirs" version = "4.0.0" @@ -5580,7 +5633,7 @@ dependencies = [ "base64 0.21.7", "bytes", "http-body-util", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-util", "pin-project", "rand 0.8.5", @@ -6197,6 +6250,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.0", "wasip2", "wasip3", ] @@ -6333,7 +6387,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tokio-retry2", - "tonic", + "tonic 0.12.3", "tower 0.4.13", "tracing", ] @@ -6346,7 +6400,7 @@ checksum = "886aa8ec755382a1fdf4651f6e6ec01f2f3bf49f2cb0f068b9a74cafd574a715" dependencies = [ "prost", "prost-types", - "tonic", + "tonic 0.12.3", ] [[package]] @@ -6750,7 +6804,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -6762,6 +6816,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.2", +] + [[package]] name = "home" version = "0.5.12" @@ -6880,7 +6943,7 @@ dependencies = [ "futures", "http 1.4.0", "http-body-util", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-rustls 0.26.0", "hyper-tls", "hyper-tungstenite", @@ -6904,6 +6967,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -6930,9 +7002,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" dependencies = [ "atomic-waker", "bytes", @@ -6945,7 +7017,6 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -6961,7 +7032,7 @@ dependencies = [ "futures-util", "headers", "http 1.4.0", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-rustls 0.27.7", "hyper-tls", "hyper-util", @@ -6981,7 +7052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ "hex", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-util", "pin-project-lite", "tokio", @@ -7013,7 +7084,7 @@ checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", "http 1.4.0", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-util", "log", "rustls 0.22.4", @@ -7031,7 +7102,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.4.0", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-util", "log", "rustls 0.23.35", @@ -7049,7 +7120,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.8.1", + "hyper 1.9.0", "hyper-util", "pin-project-lite", "tokio", @@ -7064,7 +7135,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-util", "native-tls", "tokio", @@ -7079,7 +7150,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a343d17fe7885302ed7252767dc7bb83609a874b6ff581142241ec4b73957ad" dependencies = [ "http-body-util", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-util", "pin-project-lite", "tokio", @@ -7099,7 +7170,7 @@ dependencies = [ "futures-util", "http 1.4.0", "http-body 1.0.1", - "hyper 1.8.1", + "hyper 1.9.0", "ipnet", "libc", "percent-encoding", @@ -7120,7 +7191,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-util", "pin-project-lite", "tokio", @@ -7391,9 +7462,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "inventory" -version = "0.3.22" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "009ae045c87e7082cb72dab0ccd01ae075dd00141ddc108f43a0ea150a9e7227" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" dependencies = [ "rustversion", ] @@ -7439,9 +7510,9 @@ dependencies = [ [[package]] name = "iri-string" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" dependencies = [ "memchr", "serde", @@ -7692,7 +7763,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -7784,7 +7855,7 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-http-proxy", "hyper-rustls 0.27.7", "hyper-timeout", @@ -8183,15 +8254,6 @@ dependencies = [ "hashbrown 0.15.5", ] -[[package]] -name = "lru" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f8cc7106155f10bdf99a6f379688f543ad6596a415375b36a59a054ceda1198" -dependencies = [ - "hashbrown 0.15.5", -] - [[package]] name = "lru" version = "0.16.3" @@ -8442,6 +8504,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.2", +] + [[package]] name = "md4" version = "0.10.2" @@ -8614,9 +8686,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -8720,9 +8792,9 @@ dependencies = [ [[package]] name = "mysql_async" -version = "0.36.1" +version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "277ce2f2459b2af4cc6d0a0b7892381f80800832f57c533f03e2845f4ea331ea" +checksum = "d1d9585dc9058886ff3a1f48a23024dd1d054264dee7c5ae0e4bd640c953bee5" dependencies = [ "bytes", "crossbeam-queue", @@ -8731,7 +8803,7 @@ dependencies = [ "futures-sink", "futures-util", "keyed_priority_queue", - "lru 0.14.0", + "lru 0.16.3", "mysql_common", "native-tls", "pem 3.0.6", @@ -9343,7 +9415,7 @@ dependencies = [ "http-body-util", "httparse", "humantime", - "hyper 1.8.1", + "hyper 1.9.0", "itertools 0.14.0", "md-5 0.10.6", "parking_lot", @@ -9454,7 +9526,7 @@ dependencies = [ "chrono", "dyn-clone", "ed25519-dalek", - "hmac", + "hmac 0.12.1", "http 1.4.0", "itertools 0.10.5", "log", @@ -9551,9 +9623,9 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.29.1" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e87237e2775f74896f9ad219d26a2081751187eb7c9f5c58dde20a23b95d16c" +checksum = "aaf416e4cb72756655126f7dd7bb0af49c674f4c1b9903e80c009e0c37e552e6" dependencies = [ "futures-core", "futures-sink", @@ -9565,11 +9637,11 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.27.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5feffc321035ad94088a7e5333abb4d84a8726e54a802e736ce9dd7237e85b" +checksum = "e68f63eca5fad47e570e00e893094fc17be959c80c79a7d6ec1abdd5ae6ffc16" dependencies = [ - "opentelemetry 0.27.1", + "opentelemetry 0.30.0", "tracing", "tracing-core", "tracing-subscriber", @@ -9587,6 +9659,19 @@ dependencies = [ "opentelemetry 0.27.1", ] +[[package]] +name = "opentelemetry-http" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f6639e842a97dbea8886e3439710ae463120091e2e064518ba8e716e6ac36d" +dependencies = [ + "async-trait", + "bytes", + "http 1.4.0", + "opentelemetry 0.30.0", + "reqwest 0.12.28", +] + [[package]] name = "opentelemetry-otlp" version = "0.27.0" @@ -9597,14 +9682,33 @@ dependencies = [ "futures-core", "http 1.4.0", "opentelemetry 0.27.1", - "opentelemetry-http", + "opentelemetry-http 0.27.0", "opentelemetry-proto 0.27.0", "opentelemetry_sdk 0.27.1", "prost", "serde_json", "thiserror 1.0.69", "tokio", - "tonic", + "tonic 0.12.3", + "tracing", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbee664a43e07615731afc539ca60c6d9f1a9425e25ca09c57bc36c87c55852b" +dependencies = [ + "http 1.4.0", + "opentelemetry 0.30.0", + "opentelemetry-http 0.30.0", + "opentelemetry-proto 0.30.0", + "opentelemetry_sdk 0.30.0", + "prost", + "reqwest 0.12.28", + "thiserror 2.0.18", + "tokio", + "tonic 0.13.1", "tracing", ] @@ -9619,23 +9723,22 @@ dependencies = [ "opentelemetry_sdk 0.27.1", "prost", "serde", - "tonic", + "tonic 0.12.3", ] [[package]] name = "opentelemetry-proto" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c40da242381435e18570d5b9d50aca2a4f4f4d8e146231adb4e7768023309b3" +checksum = "2e046fd7660710fe5a05e8748e70d9058dc15c94ba914e7c4faa7c728f0e8ddc" dependencies = [ "base64 0.22.1", "hex", - "opentelemetry 0.29.1", - "opentelemetry_sdk 0.29.0", + "opentelemetry 0.30.0", + "opentelemetry_sdk 0.30.0", "prost", "serde", - "tonic", - "tracing", + "tonic 0.13.1", ] [[package]] @@ -9644,6 +9747,12 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d059a296a47436748557a353c5e6c5705b9470ef6c95cfc52c21a8814ddac2" + [[package]] name = "opentelemetry_sdk" version = "0.27.1" @@ -9660,26 +9769,25 @@ dependencies = [ "rand 0.8.5", "serde_json", "thiserror 1.0.69", - "tokio", - "tokio-stream", "tracing", ] [[package]] name = "opentelemetry_sdk" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afdefb21d1d47394abc1ba6c57363ab141be19e27cc70d0e422b7f303e4d290b" +checksum = "11f644aa9e5e31d11896e024305d7e3c98a88884d9f8919dbf37a9991bc47a4b" dependencies = [ "futures-channel", "futures-executor", "futures-util", - "glob", - "opentelemetry 0.29.1", + "opentelemetry 0.30.0", "percent-encoding", "rand 0.9.0", "serde_json", "thiserror 2.0.18", + "tokio", + "tokio-stream", ] [[package]] @@ -9726,9 +9834,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "5.2.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0218004a4aae742209bee9c3cef05672f6b2708be36a50add8eb613b1f2a4008" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" dependencies = [ "num-traits", ] @@ -9926,7 +10034,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ "digest 0.10.7", - "hmac", + "hmac 0.12.1", ] [[package]] @@ -10221,7 +10329,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -10264,7 +10372,7 @@ dependencies = [ "byteorder", "bytes", "fallible-iterator 0.2.0", - "hmac", + "hmac 0.12.1", "md-5 0.10.6", "memchr", "rand 0.8.5", @@ -10274,19 +10382,19 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.10" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee9dd5fe15055d2b6806f4736aa0c9637217074e224bbec46d4041b91bb9491" +checksum = "56201207dac53e2f38e848e31b4b91616a6bb6e0c7205b77718994a7f49e70fc" dependencies = [ "base64 0.22.1", "byteorder", "bytes", "fallible-iterator 0.2.0", - "hmac", - "md-5 0.10.6", + "hmac 0.13.0", + "md-5 0.11.0", "memchr", - "rand 0.9.0", - "sha2 0.10.9", + "rand 0.10.0", + "sha2 0.11.0", "stringprep", ] @@ -10311,7 +10419,7 @@ dependencies = [ "bytes", "chrono", "fallible-iterator 0.2.0", - "postgres-protocol 0.6.10", + "postgres-protocol 0.6.11", "serde", "serde_json", "uuid", @@ -10678,7 +10786,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.1", + "rustc-hash 2.1.2", "rustls 0.23.35", "socket2 0.6.3", "thiserror 2.0.18", @@ -10699,7 +10807,7 @@ dependencies = [ "lru-slab", "rand 0.9.0", "ring 0.17.14", - "rustc-hash 2.1.1", + "rustc-hash 2.1.2", "rustls 0.23.35", "rustls-pki-types", "slab", @@ -10795,6 +10903,17 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "rand" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.0", +] + [[package]] name = "rand_chacha" version = "0.2.2" @@ -10852,6 +10971,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" + [[package]] name = "rand_distr" version = "0.5.1" @@ -11124,13 +11249,14 @@ dependencies = [ "base64 0.22.1", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2 0.4.13", "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-rustls 0.27.7", "hyper-tls", "hyper-util", @@ -11178,7 +11304,7 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-rustls 0.27.7", "hyper-util", "js-sys", @@ -11234,7 +11360,7 @@ dependencies = [ "futures", "getrandom 0.2.17", "http 1.4.0", - "hyper 1.8.1", + "hyper 1.9.0", "reqwest 0.13.1", "reqwest-middleware", "retry-policies", @@ -11265,7 +11391,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "hmac", + "hmac 0.12.1", "subtle", ] @@ -11450,7 +11576,7 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", + "const-oid 0.9.6", "digest 0.10.7", "num-bigint-dig", "num-integer", @@ -11545,9 +11671,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.40.0" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0" +checksum = "2ce901f9a19d251159075a4c37af514c3b8ef99c22e02dd8c19161cf397ee94a" dependencies = [ "arrayvec", "borsh", @@ -11558,6 +11684,7 @@ dependencies = [ "rkyv", "serde", "serde_json", + "wasm-bindgen", ] [[package]] @@ -11574,9 +11701,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustc_version" @@ -12438,7 +12565,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -12450,7 +12577,7 @@ checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.9.0", "opaque-debug", ] @@ -12462,10 +12589,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", +] + [[package]] name = "sha3" version = "0.10.8" @@ -12563,9 +12701,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "simd-json" @@ -12744,7 +12882,7 @@ dependencies = [ "data-encoding", "debugid", "if_chain", - "rustc-hash 2.1.1", + "rustc-hash 2.1.2", "serde", "serde_json", "unicode-id-start", @@ -12960,7 +13098,7 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", "md-5 0.10.6", @@ -13001,7 +13139,7 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", @@ -13747,7 +13885,7 @@ dependencies = [ "rayon", "regex", "rust-stemmers", - "rustc-hash 2.1.1", + "rustc-hash 2.1.2", "serde", "serde_json", "sketches-ddsketch", @@ -13820,7 +13958,7 @@ source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f dependencies = [ "fnv", "nom 7.1.3", - "ordered-float 5.2.0", + "ordered-float 5.3.0", "serde", "serde_json", ] @@ -14178,7 +14316,7 @@ dependencies = [ "bytes", "io-uring", "libc", - "mio 1.1.1", + "mio 1.2.0", "parking_lot", "pin-project-lite", "signal-hook-registry", @@ -14287,7 +14425,7 @@ dependencies = [ "percent-encoding", "phf 0.11.3", "pin-project-lite", - "postgres-protocol 0.6.10", + "postgres-protocol 0.6.11", "postgres-types 0.2.9", "rand 0.9.0", "socket2 0.5.10", @@ -14487,11 +14625,11 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.0+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011" +checksum = "39ca317ebc49f06bd748bfba29533eac9485569dc9bf80b849024b025e814fb9" dependencies = [ - "winnow 1.0.0", + "winnow 1.0.1", ] [[package]] @@ -14510,13 +14648,12 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.8.1", + "hyper 1.9.0", "hyper-timeout", "hyper-util", "percent-encoding", "pin-project", "prost", - "rustls-native-certs 0.8.3", "rustls-pemfile 2.2.0", "socket2 0.5.10", "tokio", @@ -14529,6 +14666,37 @@ dependencies = [ "webpki-roots 0.26.11", ] +[[package]] +name = "tonic" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "axum 0.8.4", + "base64 0.22.1", + "bytes", + "h2 0.4.13", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.9.0", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "rustls-native-certs 0.8.3", + "socket2 0.5.10", + "tokio", + "tokio-rustls 0.26.4", + "tokio-stream", + "tower 0.5.3", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -14557,7 +14725,9 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", + "indexmap 2.12.0", "pin-project-lite", + "slab", "sync_wrapper", "tokio", "tokio-util", @@ -14568,13 +14738,12 @@ dependencies = [ [[package]] name = "tower-cookies" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fd0118512cf0b3768f7fcccf0bef1ae41d68f2b45edc1e77432b36c97c56c6d" +checksum = "151b5a3e3c45df17466454bb74e9ecedecc955269bdedbf4d150dfa393b55a36" dependencies = [ - "async-trait", - "axum-core 0.4.5", - "cookie 0.18.1", + "axum-core 0.5.6", + "cookie", "futures-util", "http 1.4.0", "parking_lot", @@ -14689,14 +14858,14 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.28.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a971f6058498b5c0f1affa23e7ea202057a7301dbff68e968b2d578bcbd053" +checksum = "ddcf5959f39507d0d04d6413119c04f33b623f4f951ebcbdddddfad2d0623a9c" dependencies = [ "js-sys", "once_cell", - "opentelemetry 0.27.1", - "opentelemetry_sdk 0.27.1", + "opentelemetry 0.30.0", + "opentelemetry_sdk 0.30.0", "smallvec", "tracing", "tracing-core", @@ -14774,6 +14943,16 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" +[[package]] +name = "tree-sitter-r" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "429133cbda9f8a46e03ef3aae6abb6c3d22875f8585cad472138101bfd517255" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-ruby" version = "0.23.1" @@ -15059,9 +15238,9 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-segmentation" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da36089a805484bcccfffe0739803392c8298778a2d2f09febf76fac5ad9025b" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" [[package]] name = "unicode-width" @@ -15115,7 +15294,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] @@ -15377,6 +15556,7 @@ dependencies = [ "cfg-if", "once_cell", "rustversion", + "serde", "wasm-bindgen-macro", "wasm-bindgen-shared", ] @@ -15761,14 +15941,14 @@ dependencies = [ [[package]] name = "windmill" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-nats", "aws-config", "aws-credential-types", "aws-sdk-sqs", - "axum 0.7.9", + "axum 0.8.4", "base64 0.22.1", "chrono", "constant_time_eq 0.3.1", @@ -15778,6 +15958,8 @@ dependencies = [ "git-version", "lazy_static", "once_cell", + "opentelemetry 0.30.0", + "opentelemetry_sdk 0.30.0", "prometheus", "rand 0.9.0", "rdkafka", @@ -15837,9 +16019,9 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.665.0" +version = "1.672.0" dependencies = [ - "axum 0.7.9", + "axum 0.8.4", "chrono", "serde", "serde_json", @@ -15850,7 +16032,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "argon2", @@ -15866,14 +16048,14 @@ dependencies = [ "aws-sdk-config", "aws-sigv4", "aws-smithy-types", - "axum 0.7.9", + "axum 0.8.4", "base32", "base64 0.22.1", "bytes", "chrono", "chrono-tz", "const_format", - "cookie 0.17.0", + "cookie", "cron", "dashmap 6.1.0", "datafusion", @@ -15883,9 +16065,9 @@ dependencies = [ "futures", "git-version", "hex", - "hmac", + "hmac 0.12.1", "http 1.4.0", - "hyper 1.8.1", + "hyper 1.9.0", "indexmap 2.12.0", "itertools 0.14.0", "jsonwebtoken 8.3.0", @@ -15991,12 +16173,12 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.665.0" +version = "1.672.0" dependencies = [ - "axum 0.7.9", + "axum 0.8.4", "chrono", "http 1.4.0", - "hyper 1.8.1", + "hyper 1.9.0", "lazy_static", "quick_cache", "serde", @@ -16014,9 +16196,9 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.665.0" +version = "1.672.0" dependencies = [ - "axum 0.7.9", + "axum 0.8.4", "chrono", "serde", "serde_json", @@ -16027,10 +16209,10 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", - "axum 0.7.9", + "axum 0.8.4", "chrono", "http 1.4.0", "itertools 0.14.0", @@ -16053,7 +16235,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.665.0" +version = "1.672.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -16063,9 +16245,9 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.665.0" +version = "1.672.0" dependencies = [ - "axum 0.7.9", + "axum 0.8.4", "chrono", "itertools 0.14.0", "serde", @@ -16080,9 +16262,9 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.665.0" +version = "1.672.0" dependencies = [ - "axum 0.7.9", + "axum 0.8.4", "base64 0.22.1", "chrono", "ed25519-dalek", @@ -16103,10 +16285,10 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", - "axum 0.7.9", + "axum 0.8.4", "candle-core", "candle-nn", "candle-transformers", @@ -16126,9 +16308,9 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.665.0" +version = "1.672.0" dependencies = [ - "axum 0.7.9", + "axum 0.8.4", "chrono", "serde", "sql-builder", @@ -16142,11 +16324,11 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.665.0" +version = "1.672.0" dependencies = [ - "axum 0.7.9", + "axum 0.8.4", "chrono", - "hyper 1.8.1", + "hyper 1.9.0", "serde", "serde_json", "sql-builder", @@ -16162,9 +16344,9 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.665.0" +version = "1.672.0" dependencies = [ - "axum 0.7.9", + "axum 0.8.4", "chrono", "lazy_static", "regex", @@ -16182,9 +16364,9 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.665.0" +version = "1.672.0" dependencies = [ - "axum 0.7.9", + "axum 0.8.4", "chrono", "serde", "serde_json", @@ -16196,18 +16378,20 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-nats", "aws-config", "aws-credential-types", "aws-sdk-sqs", + "axum 0.8.4", "base64 0.22.1", "futures", "rand 0.9.0", "rdkafka", "reqwest 0.13.1", + "rmcp", "rumqttc", "serde", "serde_json", @@ -16225,14 +16409,14 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", - "axum 0.7.9", + "axum 0.8.4", "base64 0.22.1", "chrono", "http 1.4.0", - "hyper 1.8.1", + "hyper 1.9.0", "lazy_static", "serde", "serde_json", @@ -16250,9 +16434,9 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.665.0" +version = "1.672.0" dependencies = [ - "axum 0.7.9", + "axum 0.8.4", "flate2", "reqwest 0.13.1", "serde", @@ -16268,10 +16452,10 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", - "axum 0.7.9", + "axum 0.8.4", "http 1.4.0", "indexmap 2.12.0", "itertools 0.14.0", @@ -16290,9 +16474,9 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.665.0" +version = "1.672.0" dependencies = [ - "axum 0.7.9", + "axum 0.8.4", "chrono", "chrono-tz", "serde", @@ -16310,13 +16494,13 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.665.0" +version = "1.672.0" dependencies = [ - "axum 0.7.9", + "axum 0.8.4", "chrono", "futures", "http 1.4.0", - "hyper 1.8.1", + "hyper 1.9.0", "itertools 0.14.0", "lazy_static", "quick_cache", @@ -16340,10 +16524,10 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", - "axum 0.7.9", + "axum 0.8.4", "base64 0.22.1", "bytes", "chrono", @@ -16367,7 +16551,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.665.0" +version = "1.672.0" dependencies = [ "lazy_static", "serde", @@ -16379,13 +16563,14 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.665.0" +version = "1.672.0" dependencies = [ "argon2", - "axum 0.7.9", + "axum 0.8.4", "chrono", + "dashmap 6.1.0", "http 1.4.0", - "hyper 1.8.1", + "hyper 1.9.0", "lazy_static", "serde", "serde_json", @@ -16403,9 +16588,9 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.665.0" +version = "1.672.0" dependencies = [ - "axum 0.7.9", + "axum 0.8.4", "chrono", "serde", "serde_json", @@ -16417,13 +16602,13 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.665.0" +version = "1.672.0" dependencies = [ - "axum 0.7.9", + "axum 0.8.4", "chrono", "hex", "http 1.4.0", - "hyper 1.8.1", + "hyper 1.9.0", "lazy_static", "magic-crypt", "regex", @@ -16449,7 +16634,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.665.0" +version = "1.672.0" dependencies = [ "chrono", "lazy_static", @@ -16463,10 +16648,10 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", - "axum 0.7.9", + "axum 0.8.4", "k8s-openapi", "kube", "serde", @@ -16482,7 +16667,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.665.0" +version = "1.672.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -16497,7 +16682,7 @@ dependencies = [ "aws-sdk-sts", "aws-smithy-types", "aws-smithy-types-convert", - "axum 0.7.9", + "axum 0.8.4", "backon", "base64 0.22.1", "bitflags 2.9.4", @@ -16509,6 +16694,7 @@ dependencies = [ "crc", "cron", "croner", + "dashmap 6.1.0", "datafusion", "equivalent", "futures", @@ -16517,8 +16703,8 @@ dependencies = [ "git-version", "globset", "hex", - "hmac", - "hyper 1.8.1", + "hmac 0.12.1", + "hyper 1.9.0", "indexmap 2.12.0", "itertools 0.14.0", "jsonwebtoken 8.3.0", @@ -16528,11 +16714,11 @@ dependencies = [ "native-tls", "once_cell", "openidconnect", - "opentelemetry 0.27.1", + "opentelemetry 0.30.0", "opentelemetry-appender-tracing", - "opentelemetry-otlp", - "opentelemetry-semantic-conventions", - "opentelemetry_sdk 0.27.1", + "opentelemetry-otlp 0.30.0", + "opentelemetry-semantic-conventions 0.30.0", + "opentelemetry_sdk 0.30.0", "pep440_rs", "phf 0.11.3", "pin-project-lite", @@ -16565,7 +16751,7 @@ dependencies = [ "tokio-postgres 0.7.13", "tokio-stream", "tokio-util", - "tonic", + "tonic 0.13.1", "tracing", "tracing-appender", "tracing-opentelemetry", @@ -16583,7 +16769,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.665.0" +version = "1.672.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -16602,7 +16788,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.665.0" +version = "1.672.0" dependencies = [ "regex", "serde", @@ -16617,7 +16803,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -16641,7 +16827,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "futures", @@ -16658,7 +16844,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.665.0" +version = "1.672.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -16674,7 +16860,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-trait", @@ -16695,15 +16881,15 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-trait", - "axum 0.7.9", + "axum 0.8.4", "backon", "base64 0.22.1", "chrono", - "hmac", + "hmac 0.12.1", "http 1.4.0", "itertools 0.14.0", "lazy_static", @@ -16726,15 +16912,15 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-oauth2", - "axum 0.7.9", + "axum 0.8.4", "base64 0.22.1", "chrono", "hex", - "hmac", + "hmac 0.12.1", "itertools 0.14.0", "lazy_static", "reqwest 0.12.28", @@ -16750,7 +16936,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-stream", @@ -16759,7 +16945,7 @@ dependencies = [ "aws-credential-types", "aws-sdk-sts", "aws-smithy-types-convert", - "axum 0.7.9", + "axum 0.8.4", "bytes", "chrono", "datafusion", @@ -16784,7 +16970,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "futures", @@ -16802,7 +16988,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.665.0" +version = "1.672.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -16811,7 +16997,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "lazy_static", @@ -16823,7 +17009,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "serde_json", @@ -16835,7 +17021,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "gosyn", @@ -16847,7 +17033,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "lazy_static", @@ -16859,7 +17045,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "serde_json", @@ -16871,7 +17057,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "nu-parser", @@ -16882,7 +17068,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16893,7 +17079,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16905,7 +17091,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16916,7 +17102,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-recursion", @@ -16936,9 +17122,21 @@ dependencies = [ "windmill-parser", ] +[[package]] +name = "windmill-parser-r" +version = "1.672.0" +dependencies = [ + "anyhow", + "serde_json", + "tree-sitter", + "tree-sitter-r", + "wasm-bindgen", + "windmill-parser", +] + [[package]] name = "windmill-parser-ruby" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "lazy_static", @@ -16952,7 +17150,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16969,7 +17167,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "lazy_static", @@ -16982,7 +17180,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "serde", @@ -16994,7 +17192,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "lazy_static", @@ -17012,7 +17210,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -17028,7 +17226,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "rustpython-ast", @@ -17044,7 +17242,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "serde", @@ -17055,11 +17253,11 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-recursion", - "axum 0.7.9", + "axum 0.8.4", "backon", "chrono", "chrono-tz", @@ -17068,7 +17266,7 @@ dependencies = [ "futures", "futures-core", "hex", - "hmac", + "hmac 0.12.1", "itertools 0.14.0", "lazy_static", "once_cell", @@ -17092,7 +17290,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "const_format", @@ -17130,7 +17328,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.665.0" +version = "1.672.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -17141,15 +17339,15 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-recursion", - "axum 0.7.9", + "axum 0.8.4", "chrono", "futures", "http 1.4.0", - "hyper 1.8.1", + "hyper 1.9.0", "lazy_static", "quick_cache", "reqwest 0.13.1", @@ -17170,10 +17368,11 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", - "axum 0.7.9", + "async-trait", + "axum 0.8.4", "chrono", "futures", "serde", @@ -17193,14 +17392,14 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-trait", - "axum 0.7.9", + "axum 0.8.4", "chrono", "http 1.4.0", - "hyper 1.8.1", + "hyper 1.9.0", "itertools 0.14.0", "lazy_static", "rand 0.9.0", @@ -17226,11 +17425,11 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-trait", - "axum 0.7.9", + "axum 0.8.4", "base64 0.22.1", "lazy_static", "regex", @@ -17246,11 +17445,11 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-trait", - "axum 0.7.9", + "axum 0.8.4", "base64 0.22.1", "bytes", "chrono", @@ -17268,7 +17467,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-util", - "tonic", + "tonic 0.13.1", "tower-http", "tracing", "windmill-api-auth", @@ -17280,18 +17479,19 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-trait", - "axum 0.7.9", + "axum 0.8.4", "base64 0.22.1", + "chrono", "constant_time_eq 0.3.1", "futures", "hex", - "hmac", + "hmac 0.12.1", "http 1.4.0", - "hyper 1.8.1", + "hyper 1.9.0", "itertools 0.14.0", "lazy_static", "matchit 0.7.3", @@ -17315,11 +17515,11 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-trait", - "axum 0.7.9", + "axum 0.8.4", "base64 0.22.1", "itertools 0.14.0", "rdkafka", @@ -17338,11 +17538,11 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-trait", - "axum 0.7.9", + "axum 0.8.4", "base64 0.22.1", "bytes", "itertools 0.14.0", @@ -17362,12 +17562,12 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-nats", "async-trait", - "axum 0.7.9", + "axum 0.8.4", "base64 0.22.1", "itertools 0.14.0", "nkeys", @@ -17386,11 +17586,11 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-trait", - "axum 0.7.9", + "axum 0.8.4", "byteorder", "bytes", "chrono", @@ -17421,7 +17621,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-trait", @@ -17430,7 +17630,7 @@ dependencies = [ "aws-sdk-sqs", "aws-sdk-sts", "aws-smithy-types", - "axum 0.7.9", + "axum 0.8.4", "backon", "chrono", "itertools 0.14.0", @@ -17449,11 +17649,11 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-trait", - "axum 0.7.9", + "axum 0.8.4", "futures", "http 1.4.0", "itertools 0.14.0", @@ -17472,7 +17672,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "bitflags 2.9.4", @@ -17491,7 +17691,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.665.0" +version = "1.672.0" dependencies = [ "anyhow", "async-once-cell", @@ -17502,7 +17702,7 @@ dependencies = [ "aws-credential-types", "aws-sdk-bedrockruntime", "aws-smithy-types", - "axum 0.7.9", + "axum 0.8.4", "backon", "base64 0.22.1", "bit-vec 0.6.3", @@ -17519,7 +17719,7 @@ dependencies = [ "gcp_auth", "git-version", "hex", - "hmac", + "hmac 0.12.1", "hudsucker", "hyper-http-proxy", "hyper-tls", @@ -17535,8 +17735,8 @@ dependencies = [ "native-tls", "nix 0.27.1", "once_cell", - "opentelemetry 0.27.1", - "opentelemetry-proto 0.29.0", + "opentelemetry 0.30.0", + "opentelemetry-proto 0.30.0", "oracle", "pem 3.0.6", "pep440_rs", @@ -17585,6 +17785,7 @@ dependencies = [ "windmill-parser-php", "windmill-parser-py", "windmill-parser-py-imports", + "windmill-parser-r", "windmill-parser-ruby", "windmill-parser-rust", "windmill-parser-sql", @@ -17594,12 +17795,13 @@ dependencies = [ "windmill-runtime-nativets", "windmill-types", "windmill-worker-volumes", + "x509-parser 0.16.0", "yaml-rust", ] [[package]] name = "windmill-worker-volumes" -version = "1.665.0" +version = "1.672.0" dependencies = [ "bytes", "futures", @@ -18199,9 +18401,9 @@ dependencies = [ [[package]] name = "winnow" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" +checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" [[package]] name = "winsafe" @@ -18469,18 +18671,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.47" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.47" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 8204082a5b..d636d4e6f4 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.665.0" +version = "1.672.0" authors.workspace = true edition.workspace = true @@ -66,10 +66,13 @@ members = [ "./parsers/windmill-parser-nu", "./parsers/windmill-parser-java", "./parsers/windmill-parser-ruby", + "./parsers/windmill-parser-r", "./parsers/windmill-parser-bash", "./parsers/windmill-parser-py", "./parsers/windmill-parser-py-asset", "./parsers/windmill-parser-py-imports", + # Uncomment to build wasm parsers: + # "./parsers/windmill-parser-wasm", "./parsers/windmill-parser-wac", "./parsers/windmill-parser-sql", "./parsers/windmill-parser-sql-asset", @@ -79,10 +82,10 @@ members = [ "./windmill-test-utils", "./windmill-api-integration-tests", ] -exclude = ["./windmill-duckdb-ffi-internal"] +exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.665.0" +version = "1.672.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -163,7 +166,8 @@ csharp = ["windmill-worker/csharp"] nu = ["windmill-worker/nu"] java = ["windmill-worker/java"] ruby = ["windmill-worker/ruby"] -all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-kerberos", "bigquery", "csharp", "nu", "php", "java", "ruby"] +rlang = ["windmill-worker/rlang"] +all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-kerberos", "bigquery", "csharp", "nu", "php", "java", "ruby", "rlang"] # For windows we have another set of languages enabled all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-winauth", "bigquery", "csharp", "nu", "php", "java"] # Edition meta-features: shared groups @@ -260,6 +264,8 @@ windmill-dep-map.workspace = true windmill-test-utils.workspace = true windmill-worker-volumes.workspace = true windmill-types.workspace = true +opentelemetry = { workspace = true } +opentelemetry_sdk = { workspace = true } windmill-trigger.workspace = true windmill-trigger-websocket.workspace = true windmill-trigger-postgres.workspace = true @@ -345,6 +351,7 @@ windmill-parser-yaml = { path = "./parsers/windmill-parser-yaml" } windmill-parser-csharp = { path = "./parsers/windmill-parser-csharp" } windmill-parser-java = { path = "./parsers/windmill-parser-java" } windmill-parser-ruby = { path = "./parsers/windmill-parser-ruby" } +windmill-parser-r = { path = "./parsers/windmill-parser-r" } windmill-parser-nu = { path = "./parsers/windmill-parser-nu" } windmill-parser-bash = { path = "./parsers/windmill-parser-bash" } windmill-parser-sql = { path = "./parsers/windmill-parser-sql" } @@ -362,7 +369,7 @@ reqwest-middleware = { version = "^0", features = ["json"] } bitflags = "2.9.4" memchr = "2.7.4" -axum = { version = "^0.7", features = ["multipart", "macros"] } +axum = { version = "^0.8", features = ["multipart", "macros"] } headers = "^0" hyper = { version = "^1", features = ["full"] } hyper-tls = "^0.6" @@ -371,7 +378,7 @@ tokio = { version = "=1.46.1", features = ["full", "tracing", "time"] } tokio-stream = { version = "0.1.17" } tower = "^0" tower-http = { version = "^0.6", features = ["trace", "cors", "catch-panic"] } -tower-cookies = "^0.10" +tower-cookies = "^0.11" #stuck because of swc for now serde = "=1.0.220" serde_json = { version = "^1", features = ["preserve_order", "raw_value"] } @@ -386,7 +393,7 @@ tracing = "^0" tracing-subscriber = { version = "^0", features = ["env-filter", "json"] } tracing-appender = "^0" prometheus = { version = "^0", default-features = false } -cookie = { version = "0.17.0" } +cookie = { version = "0.18.0" } phf = { version = "0.11", features = ["macros"] } rust-embed = { version = "^6", features = ["interpolate-folder-path"] } mime_guess = "^2" @@ -415,6 +422,7 @@ time = "^0" serde_urlencoded = "^0" astral-tokio-tar = "^0.5.6" tempfile = "^3" +x509-parser = "^0.16" tokio-util = { version = "=0.7.17", features = ["io"] } json-pointer = "^0" itertools = "^0.14.0" @@ -566,18 +574,18 @@ flate2 = "^1" http = "^1" async-stream = "^0" -opentelemetry = "0.27.0" -tracing-opentelemetry = "0.28.0" -opentelemetry_sdk = { version = "0.27.1", features = ["rt-tokio"] } -opentelemetry-otlp = { version = "0.27.0", features = ["grpc-tonic", "tls"] } -opentelemetry-appender-tracing = "0.27.0" -opentelemetry-semantic-conventions = { version = "0.27.0", features = ["semconv_experimental"] } -opentelemetry-proto = { version = "0.29.0", features = ["with-serde", "gen-tonic"] } +opentelemetry = "0.30.0" +tracing-opentelemetry = "0.31.0" +opentelemetry_sdk = { version = "0.30.0", features = ["rt-tokio", "testing"] } +opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic", "tls"] } +opentelemetry-appender-tracing = "0.30.0" +opentelemetry-semantic-conventions = { version = "0.30.0", features = ["semconv_experimental"] } +opentelemetry-proto = { version = "0.30.0", features = ["with-serde", "gen-tonic"] } prost = "0.13" bollard = "0.18.1" -tonic = { version = "=0.12.3", features = ["tls-native-roots"] } +tonic = { version = "^0.13", features = ["tls-native-roots"] } byteorder = "1.5.0" tikv-jemallocator = { version = "0.5" } @@ -610,6 +618,7 @@ tree-sitter = { version = "0.23.0", features = [] } tree-sitter-c-sharp = "0.23.0" tree-sitter-java = "0.23.0" tree-sitter-ruby = "0.23.0" +tree-sitter-r = "1.2.0" oracle = { version = "0.6.3", features = ["chrono"] } rumqttc = { version = "0.24.0", features = ["use-native-tls"]} strum = { version = "0.27", features = ["derive"] } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index ce0a80c162..07aa65d1f5 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -6db424512b0d02f86489e85f0026581b7637d6e6 +e08a87450627bef9013498e40ee93a47bedda7ee diff --git a/backend/migrations/20260317999999_fix_trigger_superadmin_edited_by.down.sql b/backend/migrations/20260317999999_fix_trigger_superadmin_edited_by.down.sql new file mode 100644 index 0000000000..587771a417 --- /dev/null +++ b/backend/migrations/20260317999999_fix_trigger_superadmin_edited_by.down.sql @@ -0,0 +1 @@ +-- No-op: this migration is a data fixup and cannot be reversed. diff --git a/backend/migrations/20260317999999_fix_trigger_superadmin_edited_by.up.sql b/backend/migrations/20260317999999_fix_trigger_superadmin_edited_by.up.sql new file mode 100644 index 0000000000..f66c3daaa6 --- /dev/null +++ b/backend/migrations/20260317999999_fix_trigger_superadmin_edited_by.up.sql @@ -0,0 +1,48 @@ +-- Pre-fix: before permissioned_as migration drops the email column, update edited_by +-- for triggers where the user (edited_by) is not in the workspace but is a superadmin. +-- This ensures the subsequent 20260318000000 migration stores the raw email as permissioned_as +-- (via the `edited_by LIKE '%@%'` branch). +-- For instances that already applied 20260318000000, this is a no-op (email column is gone); +-- the 20260401000000 migration handles those as a fallback. + +DO $$ +DECLARE + trigger_table TEXT; + has_email BOOLEAN; +BEGIN + FOREACH trigger_table IN ARRAY ARRAY[ + 'http_trigger', + 'websocket_trigger', + 'postgres_trigger', + 'mqtt_trigger', + 'kafka_trigger', + 'nats_trigger', + 'sqs_trigger', + 'gcp_trigger', + 'email_trigger' + ] + LOOP + SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = trigger_table AND column_name = 'email' + ) INTO has_email; + + IF has_email THEN + EXECUTE format($q$ + UPDATE %I t + SET edited_by = t.email + WHERE NOT EXISTS ( + SELECT 1 FROM usr u + WHERE u.username = t.edited_by + AND u.workspace_id = t.workspace_id + ) + AND EXISTS ( + SELECT 1 FROM password p + WHERE p.email = t.email + AND p.super_admin = true + ) + $q$, trigger_table); + END IF; + END LOOP; +END; +$$; diff --git a/backend/migrations/20260326200000_service_accounts.down.sql b/backend/migrations/20260326200000_service_accounts.down.sql new file mode 100644 index 0000000000..88a50692ab --- /dev/null +++ b/backend/migrations/20260326200000_service_accounts.down.sql @@ -0,0 +1 @@ +ALTER TABLE usr DROP COLUMN is_service_account; diff --git a/backend/migrations/20260326200000_service_accounts.up.sql b/backend/migrations/20260326200000_service_accounts.up.sql new file mode 100644 index 0000000000..b9e96d0baa --- /dev/null +++ b/backend/migrations/20260326200000_service_accounts.up.sql @@ -0,0 +1 @@ +ALTER TABLE usr ADD COLUMN IF NOT EXISTS is_service_account BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/backend/migrations/20260327000000_email_varchar_255.down.sql b/backend/migrations/20260327000000_email_varchar_255.down.sql new file mode 100644 index 0000000000..b5ff4d22c2 --- /dev/null +++ b/backend/migrations/20260327000000_email_varchar_255.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE magic_link ALTER COLUMN email TYPE VARCHAR(50); +ALTER TABLE schedule ALTER COLUMN email TYPE VARCHAR(50); diff --git a/backend/migrations/20260327000000_email_varchar_255.up.sql b/backend/migrations/20260327000000_email_varchar_255.up.sql new file mode 100644 index 0000000000..95adb957b3 --- /dev/null +++ b/backend/migrations/20260327000000_email_varchar_255.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE magic_link ALTER COLUMN email TYPE VARCHAR(255); +ALTER TABLE schedule ALTER COLUMN email TYPE VARCHAR(255); diff --git a/backend/migrations/20260328000000_trigger_filter_logic.down.sql b/backend/migrations/20260328000000_trigger_filter_logic.down.sql new file mode 100644 index 0000000000..f6beac9bff --- /dev/null +++ b/backend/migrations/20260328000000_trigger_filter_logic.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE kafka_trigger DROP COLUMN filter_logic; +ALTER TABLE websocket_trigger DROP COLUMN filter_logic; diff --git a/backend/migrations/20260328000000_trigger_filter_logic.up.sql b/backend/migrations/20260328000000_trigger_filter_logic.up.sql new file mode 100644 index 0000000000..ecb99bf396 --- /dev/null +++ b/backend/migrations/20260328000000_trigger_filter_logic.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE kafka_trigger ADD COLUMN filter_logic VARCHAR(3) NOT NULL DEFAULT 'and'; +ALTER TABLE websocket_trigger ADD COLUMN filter_logic VARCHAR(3) NOT NULL DEFAULT 'and'; diff --git a/backend/migrations/20260331000000_add_rlang.down.sql b/backend/migrations/20260331000000_add_rlang.down.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/migrations/20260331000000_add_rlang.up.sql b/backend/migrations/20260331000000_add_rlang.up.sql new file mode 100644 index 0000000000..cfcb852946 --- /dev/null +++ b/backend/migrations/20260331000000_add_rlang.up.sql @@ -0,0 +1,2 @@ +ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'rlang'; +UPDATE config SET config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["rlang"]'::jsonb) WHERE name = 'worker__default' AND config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php", "rust", "ansible", "csharp", "nu", "java", "duckdb", "ruby"]}'::jsonb AND NOT config->'worker_tags' @> '"rlang"'::jsonb; diff --git a/backend/parsers/windmill-parser-r/Cargo.toml b/backend/parsers/windmill-parser-r/Cargo.toml new file mode 100644 index 0000000000..42701f9c12 --- /dev/null +++ b/backend/parsers/windmill-parser-r/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "windmill-parser-r" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser_r" +path = "./src/lib.rs" + +[dependencies] +windmill-parser.workspace = true +tree-sitter.workspace = true +tree-sitter-r.workspace = true +anyhow.workspace = true +wasm-bindgen.workspace = true +serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-r/src/lib.rs b/backend/parsers/windmill-parser-r/src/lib.rs new file mode 100644 index 0000000000..4d018fa8cc --- /dev/null +++ b/backend/parsers/windmill-parser-r/src/lib.rs @@ -0,0 +1,363 @@ +#![cfg_attr(target_arch = "wasm32", feature(c_variadic))] + +#[cfg(target_arch = "wasm32")] +pub mod wasm_libc; + +use anyhow::anyhow; +use serde_json::Value; +use tree_sitter::Node; +use tree_sitter::Range; +use windmill_parser::json_to_typ; +use windmill_parser::Arg; +use windmill_parser::MainArgSignature; + +pub fn parse_r_sig_meta(code: &str) -> anyhow::Result { + let mut parser = tree_sitter::Parser::new(); + let language = tree_sitter_r::LANGUAGE; + parser + .set_language(&language.into()) + .map_err(|e| anyhow!("Error setting R as language: {e}"))?; + + let tree = parser + .parse(code, None) + .ok_or(anyhow!("Failed to parse code"))?; + let root_node = tree.root_node(); + + let args = find_main_signature(root_node, code)?; + let main_sig = MainArgSignature { + star_args: false, + star_kwargs: false, + args: args.unwrap_or_default(), + has_preprocessor: None, + auto_kind: None, + }; + + Ok(main_sig) +} + +pub fn parse_r_signature(code: &str) -> anyhow::Result { + Ok(parse_r_sig_meta(code)?) +} + +/// Extract package names from `library(...)` and `require(...)` calls in R code. +/// Returns a newline-separated list of package names. +pub fn parse_r_requirements(code: &str) -> anyhow::Result { + let mut parser = tree_sitter::Parser::new(); + let language = tree_sitter_r::LANGUAGE; + parser + .set_language(&language.into()) + .map_err(|e| anyhow!("Error setting R as language: {e}"))?; + + let tree = parser + .parse(code, None) + .ok_or(anyhow!("Failed to parse code"))?; + let root_node = tree.root_node(); + + let mut packages = vec![]; + find_library_calls(root_node, code, &mut packages); + + // Deduplicate and exclude base packages + packages.sort(); + packages.dedup(); + packages.retain(|p| !is_base_package(p)); + + Ok(packages.join("\n")) +} + +fn find_library_calls(node: Node, code: &str, packages: &mut Vec) { + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + if child.kind() == "call" { + // call node: child 0 is the function name, child 1 is arguments + if let (Some(func_node), Some(args_node)) = (child.child(0), child.child(1)) { + let func_name = func_node.utf8_text(code.as_bytes()).unwrap_or(""); + if func_name == "library" || func_name == "require" { + // AST: arguments → ( + argument → identifier/string + ) + if args_node.kind() == "arguments" { + let mut args_cursor = args_node.walk(); + for arg in args_node.children(&mut args_cursor) { + if arg.kind() == "argument" { + // The argument node wraps the actual value + if let Some(value_node) = arg.child(0) { + let pkg = value_node + .utf8_text(code.as_bytes()) + .unwrap_or("") + .trim_matches('"') + .trim_matches('\''); + if !pkg.is_empty() { + packages.push(pkg.to_string()); + } + } + break; // only first arg + } + } + } + } + } + } + // Recurse into children to find nested library() calls + find_library_calls(child, code, packages); + } +} + +fn is_base_package(pkg: &str) -> bool { + matches!( + pkg, + "base" + | "compiler" + | "datasets" + | "grDevices" + | "graphics" + | "grid" + | "methods" + | "parallel" + | "splines" + | "stats" + | "stats4" + | "tcltk" + | "tools" + | "utils" + ) +} + +/// Find the main function signature in R code. +/// R function definitions look like: `main <- function(x, y = 10) { ... }` +/// In the tree-sitter-r AST, this is a `binary_operator` node with: +/// - child 0: identifier "main" +/// - child 1: "<-" or "=" +/// - child 2: function_definition node +fn find_main_signature<'a>(root_node: Node<'a>, code: &str) -> anyhow::Result>> { + let mut cursor = root_node.walk(); + for x in root_node.children(&mut cursor) { + if x.kind() == "binary_operator" { + let child_count = x.child_count(); + if child_count < 3 { + continue; + } + + // First child should be identifier "main" + let ident_node = x.child(0).unwrap(); + if ident_node.kind() != "identifier" { + continue; + } + let ident = ident_node.utf8_text(code.as_bytes()).unwrap_or(""); + if ident != "main" { + continue; + } + + // Second child should be "<-" or "=" + let op_node = x.child(1).unwrap(); + let op = op_node.utf8_text(code.as_bytes()).unwrap_or(""); + if op != "<-" && op != "=" { + continue; + } + + // Third child should be the function_definition + let func_node = x.child(2).unwrap(); + if func_node.kind() != "function_definition" { + continue; + } + + return Ok(Some(parse_function_params(func_node, code)?)); + } + } + Ok(None) +} + +/// Parse parameters from a function_definition node. +/// function_definition has children: "function", parameters, body +/// Each parameter node has: +/// - 1 child (identifier) for positional args +/// - 3 children (identifier, "=", value) for default args +fn parse_function_params(func_node: Node, code: &str) -> anyhow::Result> { + let mut args = vec![]; + let mut func_cursor = func_node.walk(); + + for child in func_node.children(&mut func_cursor) { + if child.kind() == "parameters" { + let mut param_cursor = child.walk(); + for param in child.children(&mut param_cursor) { + if param.kind() != "parameter" { + continue; + } + + let param_child_count = param.child_count(); + if param_child_count == 1 { + // Simple parameter: just identifier + let ident_node = param.child(0).unwrap(); + let name = ident_node.utf8_text(code.as_bytes())?; + args.push(Arg { name: name.to_owned(), ..Default::default() }); + } else if param_child_count >= 3 { + // Default parameter: identifier = value + let ident_node = param.child(0).unwrap(); + let value_node = param.child(2).unwrap(); + let name = ident_node.utf8_text(code.as_bytes())?; + + let Range { start_byte, end_byte, .. } = value_node.range(); + let raw = &code[start_byte..end_byte]; + // Convert R literals to JSON + let unparsed = raw + .replace("NULL", "null") + .replace("TRUE", "true") + .replace("FALSE", "false"); + match serde_json::from_str::(&unparsed) { + Ok(default) => { + args.push(Arg { + name: name.to_owned(), + typ: json_to_typ(&default, true), + default: Some(default), + has_default: true, + ..Default::default() + }); + } + Err(_) => { + args.push(Arg { + name: name.to_owned(), + has_default: true, + ..Default::default() + }); + } + } + } + } + } + } + Ok(args) +} + +#[cfg(test)] +mod test { + use serde_json::json; + use windmill_parser::Typ; + + use super::parse_r_sig_meta as parse; + + #[test] + fn test_parse_r_no_main() { + let code = r#" +not_main <- function() {} +helper <- function(x) { x + 1 } +"#; + let sig = parse(code).unwrap(); + assert_eq!( + sig, + windmill_parser::MainArgSignature { auto_kind: None, ..Default::default() } + ); + } + + #[test] + fn test_parse_r_no_args() { + let code = r#" +main <- function() { + return(42) +} +"#; + let sig = parse(code).unwrap(); + assert_eq!( + sig, + windmill_parser::MainArgSignature { auto_kind: None, ..Default::default() } + ); + } + + #[test] + fn test_parse_r_positional_args() { + let code = r#"main <- function(a, b, c) { a + b + c }"#; + let sig = parse(code).unwrap(); + assert_eq!( + sig, + windmill_parser::MainArgSignature { + args: vec![ + windmill_parser::Arg { name: "a".into(), ..Default::default() }, + windmill_parser::Arg { name: "b".into(), ..Default::default() }, + windmill_parser::Arg { name: "c".into(), ..Default::default() }, + ], + auto_kind: None, + ..Default::default() + } + ); + } + + #[test] + fn test_parse_r_default_args() { + let code = r#"main <- function(a = 10, b = "hey", c = FALSE) { }"#; + let sig = parse(code).unwrap(); + assert_eq!(sig.args.len(), 3); + assert_eq!(sig.args[0].name, "a"); + assert_eq!(sig.args[0].default, Some(json!(10))); + assert_eq!(sig.args[0].typ, Typ::Int); + assert_eq!(sig.args[1].name, "b"); + assert_eq!(sig.args[1].default, Some(json!("hey"))); + assert_eq!(sig.args[1].typ, Typ::Str(None)); + assert_eq!(sig.args[2].name, "c"); + assert_eq!(sig.args[2].default, Some(json!(false))); + assert_eq!(sig.args[2].typ, Typ::Bool); + } + + #[test] + fn test_parse_r_equals_assignment() { + let code = r#"main = function(x, y = 5) { x + y }"#; + let sig = parse(code).unwrap(); + assert_eq!(sig.args.len(), 2); + assert_eq!(sig.args[0].name, "x"); + assert_eq!(sig.args[1].name, "y"); + assert_eq!(sig.args[1].default, Some(json!(5))); + } + + #[test] + fn test_parse_r_null_default() { + let code = r#"main <- function(x = NULL) { x }"#; + let sig = parse(code).unwrap(); + assert_eq!(sig.args.len(), 1); + assert_eq!(sig.args[0].name, "x"); + assert_eq!(sig.args[0].default, Some(json!(null))); + } + + #[test] + fn test_parse_r_requirements() { + use super::parse_r_requirements; + + let code = r#" +library(dplyr) +library(ggplot2) +require(tidyr) +library(stats) + +main <- function(x) { + library(stringr) + x +} +"#; + let reqs = parse_r_requirements(code).unwrap(); + let pkgs: Vec<&str> = reqs.lines().collect(); + assert!(pkgs.contains(&"dplyr")); + assert!(pkgs.contains(&"ggplot2")); + assert!(pkgs.contains(&"tidyr")); + assert!(pkgs.contains(&"stringr")); + assert!(!pkgs.contains(&"stats")); // base package excluded + } + + #[test] + fn test_parse_r_requirements_string_args() { + use super::parse_r_requirements; + + let code = r#" +library("data.table") +require("jsonlite") + +main <- function() { } +"#; + let reqs = parse_r_requirements(code).unwrap(); + let pkgs: Vec<&str> = reqs.lines().collect(); + assert!(pkgs.contains(&"data.table")); + assert!(pkgs.contains(&"jsonlite")); + } + + #[test] + fn test_parse_r_requirements_no_deps() { + use super::parse_r_requirements; + + let code = r#"main <- function(x) { x + 1 }"#; + let reqs = parse_r_requirements(code).unwrap(); + assert!(reqs.is_empty()); + } +} diff --git a/backend/parsers/windmill-parser-r/src/wasm_libc.rs b/backend/parsers/windmill-parser-r/src/wasm_libc.rs new file mode 100644 index 0000000000..924748a073 --- /dev/null +++ b/backend/parsers/windmill-parser-r/src/wasm_libc.rs @@ -0,0 +1,293 @@ +use std::collections::BTreeMap; +use std::sync::{Mutex, OnceLock}; +use std::{ + alloc::{self, Layout}, + ffi::{c_char, c_int, c_void}, + mem::align_of, + ptr, +}; +use wasm_bindgen::prelude::*; + +/* -------------------------------- stdlib.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn abort() { + panic!("Aborted from C"); +} + +macro_rules! console_log { + ($($t:tt)*) => (unsafe { log(&format_args!($($t)*).to_string()) }) +} + +#[wasm_bindgen] +extern "C" { + #[wasm_bindgen(js_namespace = console)] + fn log(a: &str); +} + +#[no_mangle] +pub unsafe extern "C" fn malloc(size: usize) -> *mut c_void { + if size == 0 { + return ptr::null_mut(); + } + + let (layout, offset_to_data) = layout_for_size_prepended(size); + let buf = alloc::alloc(layout); + store_layout(buf, layout, offset_to_data) +} + +#[no_mangle] +pub unsafe extern "C" fn calloc(count: usize, size: usize) -> *mut c_void { + if count == 0 || size == 0 { + return ptr::null_mut(); + } + + let (layout, offset_to_data) = layout_for_size_prepended(size * count); + let buf = alloc::alloc_zeroed(layout); + store_layout(buf, layout, offset_to_data) +} + +#[no_mangle] +pub unsafe extern "C" fn realloc(buf: *mut c_void, new_size: usize) -> *mut c_void { + if buf.is_null() { + malloc(new_size) + } else if new_size == 0 { + free(buf); + ptr::null_mut() + } else { + let (old_buf, old_layout) = retrieve_layout(buf); + let (new_layout, offset_to_data) = layout_for_size_prepended(new_size); + let new_buf = alloc::realloc(old_buf, old_layout, new_layout.size()); + store_layout(new_buf, new_layout, offset_to_data) + } +} + +#[no_mangle] +pub unsafe extern "C" fn free(buf: *mut c_void) { + if buf.is_null() { + return; + } + let (buf, layout) = retrieve_layout(buf); + alloc::dealloc(buf, layout); +} + +// In all these allocations, we store the layout before the data for later retrieval. +// This is because we need to know the layout when deallocating the memory. +// Here are some helper methods for that: + +/// Given a pointer to the data, retrieve the layout and the pointer to the layout. +unsafe fn retrieve_layout(buf: *mut c_void) -> (*mut u8, Layout) { + let (_, layout_offset) = Layout::new::() + .extend(Layout::from_size_align(0, align_of::<*const u8>() * 2).unwrap()) + .unwrap(); + + let buf = (buf as *mut u8).offset(-(layout_offset as isize)); + let layout = *(buf as *mut Layout); + + (buf, layout) +} + +/// Calculate a layout for a given size with space for storing a layout at the start. +/// Returns the layout and the offset to the data. +fn layout_for_size_prepended(size: usize) -> (Layout, usize) { + Layout::new::() + .extend(Layout::from_size_align(size, align_of::<*const u8>() * 2).unwrap()) + .unwrap() +} + +/// Store a layout in the pointer, returning a pointer to where the data should be stored. +unsafe fn store_layout(buf: *mut u8, layout: Layout, offset_to_data: usize) -> *mut c_void { + *(buf as *mut Layout) = layout; + (buf as *mut u8).offset(offset_to_data as isize) as *mut c_void +} + +/* -------------------------------- string.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn strncmp(ptr1: *const c_void, ptr2: *const c_void, n: usize) -> c_int { + let s1 = std::slice::from_raw_parts(ptr1 as *const u8, n); + let s2 = std::slice::from_raw_parts(ptr2 as *const u8, n); + + for (a, b) in s1.iter().zip(s2.iter()) { + if *a != *b || *a == 0 { + return (*a as i32) - (*b as i32); + } + } + + 0 +} + +// Implementation by AI: +pub type size_t = usize; +use std::slice; +#[no_mangle] +pub unsafe extern "C" fn memchr(haystack: *const c_void, needle: c_int, len: usize) -> *mut c_void { + if haystack.is_null() || len == 0 { + return ptr::null_mut(); // Return null if the input pointer is null or length is zero + } + + let needle_byte = needle as u8; // Convert needle to a byte + + // Create a pointer to the start of the haystack + let mut current = haystack as *const u8; + + // Iterate through the memory block + for _ in 0..len { + if *current == needle_byte { + return current as *mut c_void; // Return the pointer to the found byte + } + current = current.add(1); // Move to the next byte + } + + ptr::null_mut() // Return null if the byte was not found +} + +#[no_mangle] +pub unsafe extern "C" fn strchr(mut s: *const c_char, c: c_int) -> *mut c_char { + if s.is_null() { + return std::ptr::null_mut(); // Return null if the input string is null + } + + let target = c as u8 as char; // Convert c to a char + let mut current = s; + + // Iterate through the string until we find the character or reach the end + while *current != 0 { + if *current as u8 as char == target { + return current as *mut c_char; // Return the pointer to the found character + } + current = current.add(1); // Move to the next character + } + + std::ptr::null_mut() // Return null if the character was not found +} +// End of AI implemetation +/* -------------------------------- wctype.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn iswspace(c: c_int) -> bool { + char::from_u32(c as u32).map_or(false, |c| c.is_whitespace()) +} + +#[no_mangle] +pub unsafe extern "C" fn iswalnum(c: c_int) -> bool { + char::from_u32(c as u32).map_or(false, |c| c.is_alphanumeric()) +} + +// Implementation by AI: +pub type wint_t = u32; + +#[no_mangle] +pub extern "C" fn iswdigit(wc: wint_t) -> c_int { + // Check if the character is a digit ('0' to '9') + if wc >= '0' as wint_t && wc <= '9' as wint_t { + return 1; // Return true (1) + } + 0 // Return false (0) +} + +#[no_mangle] +pub extern "C" fn iswupper(wc: wint_t) -> c_int { + // Check if the character is an uppercase letter ('A' to 'Z') + if wc >= 'A' as wint_t && wc <= 'Z' as wint_t { + return 1; // Return true (1) + } + 0 // Return false (0) +} + +#[no_mangle] +pub extern "C" fn iswalpha(wc: wint_t) -> c_int { + // Check if the character is an alphabetic character ('A' to 'Z' or 'a' to 'z') + if (wc >= 'A' as wint_t && wc <= 'Z' as wint_t) || (wc >= 'a' as wint_t && wc <= 'z' as wint_t) + { + return 1; // Return true (1) + } + 0 // Return false (0) +} + +#[no_mangle] +pub extern "C" fn iswlower(wc: wint_t) -> c_int { + // Check if the character is a lowercase letter ('a' to 'z') + if wc >= 'a' as wint_t && wc <= 'z' as wint_t { + return 1; // Return true (1) + } + 0 // Return false (0) +} +// End of AI implemetation + +/* --------------------------------- time.h --------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn clock() -> u64 { + panic!("clock is not supported"); +} + +/* --------------------------------- ctype.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn isprint(c: c_int) -> bool { + c >= 32 && c <= 126 +} + +/* --------------------------------- stdio.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn fprintf(_file: *mut c_void, _format: *const c_void, _args: ...) -> c_int { + panic!("fprintf is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fputs(_s: *const c_void, _file: *mut c_void) -> c_int { + panic!("fputs is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fputc(_c: c_int, _file: *mut c_void) -> c_int { + panic!("fputc is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fdopen(_fd: c_int, _mode: *const c_void) -> *mut c_void { + panic!("fdopen is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fclose(_file: *mut c_void) -> c_int { + panic!("fclose is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fwrite( + _ptr: *const c_void, + _size: usize, + _nmemb: usize, + _stream: *mut c_void, +) -> usize { + panic!("fwrite is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn vsnprintf( + _buf: *mut c_char, + _size: usize, + _format: *const c_char, + _args: ... +) -> c_int { + panic!("vsnprintf is not supported"); +} + +#[no_mangle] +pub extern "C" fn clock_gettime(ptr: usize, new_size: usize) { + panic!("clock_gettime is not supported"); +} + +// int snprintf( char* restrict buffer, size_t bufsz, const char* restrict format, ... ); +#[no_mangle] +pub extern "C" fn snprintf() { + panic!("snprintf is not supported"); +} + +#[no_mangle] +pub extern "C" fn __assert_fail(_: *const i32, _: *const i32, _: *const i32, _: *const i32) { + panic!("oh no"); +} diff --git a/backend/parsers/windmill-parser-wac/src/dag.rs b/backend/parsers/windmill-parser-wac/src/dag.rs index 662f5a06b6..f70bd6ac53 100644 --- a/backend/parsers/windmill-parser-wac/src/dag.rs +++ b/backend/parsers/windmill-parser-wac/src/dag.rs @@ -27,11 +27,15 @@ pub struct DagNode { #[serde(tag = "type")] pub enum DagNodeType { Step { name: String, script: String }, + InlineStep { name: String }, + Sleep { seconds: String }, + WaitForApproval, Branch { condition_source: String }, ParallelStart, ParallelEnd, LoopStart { iter_source: String }, LoopEnd, + Merge, Return, } diff --git a/backend/parsers/windmill-parser-wac/src/python.rs b/backend/parsers/windmill-parser-wac/src/python.rs index 74f0117376..f91067b711 100644 --- a/backend/parsers/windmill-parser-wac/src/python.rs +++ b/backend/parsers/windmill-parser-wac/src/python.rs @@ -37,7 +37,8 @@ impl LineIndex { /// Maps task function name → optional external path (from `@task(path="...")`) type TaskFunctions = HashMap>; -/// First pass: scan top-level `@task async def foo(...)` declarations. +/// First pass: scan top-level `@task async def foo(...)` declarations +/// and `foo = task_script("path")` / `foo = task_flow("path")` assignments. fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { let mut tasks = HashMap::new(); for stmt in stmts { @@ -61,6 +62,30 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { } } } + // foo = task_script("path") or foo = task_flow("path") + if let Stmt::Assign(assign) = stmt { + if let Expr::Call(call) = assign.value.as_ref() { + if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() { + if id.as_str() == "task_script" || id.as_str() == "task_flow" { + // Extract the path from the first positional argument + let path = call.args.first().and_then(|arg| { + if let Expr::Constant(c) = arg { + if let rustpython_parser::ast::Constant::Str(s) = &c.value { + return Some(s.to_string()); + } + } + None + }); + // Extract variable name from target + if let Some(Expr::Name(ExprName { id: var_name, .. })) = + assign.targets.first() + { + tasks.insert(var_name.to_string(), path); + } + } + } + } + } } tasks } @@ -88,8 +113,6 @@ struct WacWalker { node_counter: usize, line_index: LineIndex, task_functions: TaskFunctions, - in_try: bool, - in_while: bool, in_nested_func: bool, in_comprehension: bool, } @@ -103,8 +126,6 @@ impl WacWalker { node_counter: 0, line_index: LineIndex::new(source), task_functions, - in_try: false, - in_while: false, in_nested_func: false, in_comprehension: false, } @@ -292,6 +313,9 @@ impl WacWalker { if self.is_task_fn_call(expr) { return true; } + if Self::is_sdk_call(expr) { + return true; + } match expr { Expr::Await(ExprAwait { value, .. }) => self.expr_contains_step(value), Expr::Call(call) => { @@ -307,6 +331,17 @@ impl WacWalker { } } + /// Check if expr is a call to a known SDK function (step, sleep, wait_for_approval) + fn is_sdk_call(expr: &Expr) -> bool { + if let Expr::Call(call) = expr { + if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() { + let name = id.as_str(); + return name == "step" || name == "sleep" || name == "wait_for_approval"; + } + } + false + } + /// Walk a list of statements, returning (first_node_id, last_node_id) fn walk_body(&mut self, body: &[Stmt]) -> Option<(String, String)> { let mut first_id: Option = None; @@ -353,13 +388,17 @@ impl WacWalker { } fn walk_expr_stmt(&mut self, expr: &Expr) -> Option<(String, String)> { - // await task_fn(...) + // await task_fn(...) / await step(...) / await sleep(...) / await wait_for_approval(...) if let Expr::Await(ExprAwait { value, .. }) = expr { // await task_fn(...) if let Expr::Call(call) = value.as_ref() { if self.is_task_fn_call(&Expr::Call(call.clone())) { return self.emit_step(call, expr); } + // Check for SDK-level calls: step(), sleep(), wait_for_approval() + if let Some(result) = self.try_emit_sdk_call(call, expr) { + return Some(result); + } } // await asyncio.gather(task_fn(...), task_fn(...), ...) if Self::is_asyncio_gather_call(value) { @@ -378,17 +417,69 @@ impl WacWalker { None } + /// Try to emit a node for SDK-level calls: step(), sleep(), wait_for_approval() + fn try_emit_sdk_call(&mut self, call: &ExprCall, expr: &Expr) -> Option<(String, String)> { + let callee_name = match call.func.as_ref() { + Expr::Name(ExprName { id, .. }) => Some(id.as_str()), + _ => None, + }?; + + let line = self.line_of_expr(expr); + + match callee_name { + "step" => { + // step("name", fn) — extract the name from the first string argument + let name = call + .args + .first() + .and_then(|arg| { + if let Expr::Constant(c) = arg { + if let rustpython_parser::ast::Constant::Str(s) = &c.value { + return Some(s.to_string()); + } + } + None + }) + .unwrap_or_else(|| "step".to_string()); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::InlineStep { name: name.clone() }, + label: name, + line, + }); + Some((node_id.clone(), node_id)) + } + "sleep" => { + let seconds = call + .args + .first() + .map(|arg| Self::expr_to_source(arg)) + .unwrap_or_else(|| "?".to_string()); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::Sleep { seconds: seconds.clone() }, + label: format!("sleep({seconds})"), + line, + }); + Some((node_id.clone(), node_id)) + } + "wait_for_approval" => { + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::WaitForApproval, + label: "wait_for_approval".to_string(), + line, + }); + Some((node_id.clone(), node_id)) + } + _ => None, + } + } + fn emit_step(&mut self, call: &ExprCall, expr: &Expr) -> Option<(String, String)> { - if self.in_try { - self.errors - .push(validation::error_step_in_try(self.line_of_expr(expr))); - return None; - } - if self.in_while { - self.errors - .push(validation::error_step_in_while(self.line_of_expr(expr))); - return None; - } if self.in_nested_func { self.errors.push(validation::error_step_in_nested_function( self.line_of_expr(expr), @@ -416,17 +507,6 @@ impl WacWalker { } fn emit_parallel(&mut self, gather_call: &ExprCall, expr: &Expr) -> Option<(String, String)> { - if self.in_try { - self.errors - .push(validation::error_step_in_try(self.line_of_expr(expr))); - return None; - } - if self.in_while { - self.errors - .push(validation::error_step_in_while(self.line_of_expr(expr))); - return None; - } - let line = self.line_of_expr(expr); let start_id = self.next_id(); let start_node_id = self.add_node(DagNode { @@ -491,8 +571,6 @@ impl WacWalker { line, }); - let merge_id = format!("{branch_id}_merge"); - let mut last_ids = Vec::new(); if let Some((true_first, true_last)) = self.walk_body(&if_stmt.body) { @@ -514,7 +592,17 @@ impl WacWalker { if last_ids.len() == 1 { Some((branch_node_id, last_ids.into_iter().next().unwrap())) } else { - Some((branch_node_id, merge_id)) + let merge_id = format!("{branch_id}_merge"); + let merge_node_id = self.add_node(DagNode { + id: merge_id, + node_type: DagNodeType::Merge, + label: "merge".to_string(), + line, + }); + for last in last_ids { + self.add_edge(&last, &merge_node_id, None); + } + Some((branch_node_id, merge_node_id)) } } @@ -552,11 +640,36 @@ impl WacWalker { } fn walk_while(&mut self, while_stmt: &StmtWhile) -> Option<(String, String)> { - if self.body_contains_step(&while_stmt.body) { - let line = self.line_index.line_of(while_stmt.range.start().to_usize()); - self.errors.push(validation::error_step_in_while(line)); + if !self.body_contains_step(&while_stmt.body) { + return None; } - None + + let line = self.line_index.line_of(while_stmt.range.start().to_usize()); + let condition = Self::expr_to_source(&while_stmt.test); + + let start_id = self.next_id(); + let start_node_id = self.add_node(DagNode { + id: start_id.clone(), + node_type: DagNodeType::LoopStart { iter_source: condition }, + label: "while".to_string(), + line, + }); + + if let Some((body_first, body_last)) = self.walk_body(&while_stmt.body) { + self.add_edge(&start_node_id, &body_first, None); + self.add_edge(&body_last, &start_node_id, Some("next".to_string())); + } + + let end_id = self.next_id(); + let end_node_id = self.add_node(DagNode { + id: end_id.clone(), + node_type: DagNodeType::LoopEnd, + label: "end while".to_string(), + line, + }); + self.add_edge(&start_node_id, &end_node_id, Some("done".to_string())); + + Some((start_node_id, end_node_id)) } fn walk_try(&mut self, try_stmt: &StmtTry) -> Option<(String, String)> { @@ -569,11 +682,17 @@ impl WacWalker { } }); - if has_steps { - let line = self.line_index.line_of(try_stmt.range.start().to_usize()); - self.errors.push(validation::error_step_in_try(line)); + if !has_steps { + return None; } - None + + let line = self.line_index.line_of(try_stmt.range.start().to_usize()); + self.emit_try_catch_branch( + &try_stmt.body, + &try_stmt.handlers, + &try_stmt.finalbody, + line, + ) } fn walk_try_star(&mut self, try_stmt: &StmtTryStar) -> Option<(String, String)> { @@ -586,11 +705,81 @@ impl WacWalker { } }); - if has_steps { - let line = self.line_index.line_of(try_stmt.range.start().to_usize()); - self.errors.push(validation::error_step_in_try(line)); + if !has_steps { + return None; } - None + + let line = self.line_index.line_of(try_stmt.range.start().to_usize()); + self.emit_try_catch_branch( + &try_stmt.body, + &try_stmt.handlers, + &try_stmt.finalbody, + line, + ) + } + + fn emit_try_catch_branch( + &mut self, + try_body: &[Stmt], + handlers: &[rustpython_parser::ast::ExceptHandler], + finally_body: &[Stmt], + line: usize, + ) -> Option<(String, String)> { + let branch_id = self.next_id(); + let branch_node_id = self.add_node(DagNode { + id: branch_id.clone(), + node_type: DagNodeType::Branch { condition_source: "try/except".to_string() }, + label: "try".to_string(), + line, + }); + + let mut last_ids = Vec::new(); + + // Try body + if let Some((try_first, try_last)) = self.walk_body(try_body) { + self.add_edge(&branch_node_id, &try_first, Some("try".to_string())); + last_ids.push(try_last); + } else { + last_ids.push(branch_node_id.clone()); + } + + // Except handlers + for handler in handlers { + match handler { + rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + if let Some((catch_first, catch_last)) = self.walk_body(&eh.body) { + self.add_edge(&branch_node_id, &catch_first, Some("except".to_string())); + last_ids.push(catch_last); + } + } + } + } + + // Finally body — sequential after merge + let merge_last = if last_ids.len() == 1 { + last_ids.into_iter().next().unwrap() + } else { + let merge_id = format!("{branch_id}_merge"); + let merge_node_id = self.add_node(DagNode { + id: merge_id, + node_type: DagNodeType::Merge, + label: "merge".to_string(), + line, + }); + for last in last_ids { + self.add_edge(&last, &merge_node_id, None); + } + merge_node_id + }; + + if !finally_body.is_empty() { + if let Some((finally_first, finally_last)) = self.walk_body(finally_body) { + self.add_edge(&merge_last, &finally_first, None); + return Some((branch_node_id, finally_last)); + } + } + + Some((branch_node_id, merge_last)) } fn walk_return(&mut self, ret: &StmtReturn) -> Option<(String, String)> { diff --git a/backend/parsers/windmill-parser-wac/src/typescript.rs b/backend/parsers/windmill-parser-wac/src/typescript.rs index bd777749ea..87fa7cef01 100644 --- a/backend/parsers/windmill-parser-wac/src/typescript.rs +++ b/backend/parsers/windmill-parser-wac/src/typescript.rs @@ -51,22 +51,29 @@ fn extract_var_name(pat: &Pat) -> Option { } } -/// Check if expr is `task(async fn)` or `task("path", async fn)`. -/// Returns Some(optional_path) if it is a task() call. +/// Check if expr is `task(async fn)`, `task("path", async fn)`, +/// `taskScript("path")`, or `taskFlow("path")`. +/// Returns Some(optional_path) if it is a task/taskScript/taskFlow call. fn extract_task_call_info(expr: &Expr) -> Option> { if let Expr::Call(call) = expr { if let Callee::Expr(callee) = &call.callee { if let Expr::Ident(ident) = callee.as_ref() { - if ident.sym.as_ref() == "task" { + let name = ident.sym.as_ref(); + if name == "task" { // task("f/path", async fn) or task(async fn) if call.args.len() == 2 { - // task("f/path", async fn) let path = extract_string_lit(&call.args[0].expr); return Some(path); } else if call.args.len() == 1 { - // task(async fn) return Some(None); } + } else if name == "taskScript" || name == "taskFlow" { + // taskScript("./helper.ts") or taskFlow("f/my_flow") + if let Some(first_arg) = call.args.first() { + let path = extract_string_lit(&first_arg.expr); + return Some(path); + } + return Some(None); } } } @@ -81,8 +88,6 @@ struct TsWacWalker { node_counter: usize, cm: Lrc, task_functions: TaskFunctions, - in_try: bool, - in_while: bool, in_nested_func: bool, } @@ -95,8 +100,6 @@ impl TsWacWalker { node_counter: 0, cm, task_functions, - in_try: false, - in_while: false, in_nested_func: false, } } @@ -224,6 +227,9 @@ impl TsWacWalker { if self.is_task_call(expr) { return true; } + if Self::is_sdk_call(expr) { + return true; + } match expr { Expr::Await(await_expr) => self.expr_contains_step(&await_expr.arg), Expr::Call(call) => { @@ -237,6 +243,19 @@ impl TsWacWalker { } } + /// Check if expr is a call to a known SDK function (step, sleep, waitForApproval) + fn is_sdk_call(expr: &Expr) -> bool { + if let Expr::Call(call) = expr { + if let Callee::Expr(callee) = &call.callee { + if let Expr::Ident(ident) = callee.as_ref() { + let name = ident.sym.as_ref(); + return name == "step" || name == "sleep" || name == "waitForApproval"; + } + } + } + false + } + fn walk_body(&mut self, stmts: &[Stmt]) -> Option<(String, String)> { let mut first_id: Option = None; let mut prev_id: Option = None; @@ -294,12 +313,16 @@ impl TsWacWalker { } fn walk_expr_stmt(&mut self, expr: &Expr) -> Option<(String, String)> { - // await task_fn(...) + // await task_fn(...) / await step(...) / await sleep(...) / await waitForApproval(...) if let Expr::Await(await_expr) = expr { if let Expr::Call(call) = await_expr.arg.as_ref() { if self.is_task_call(&Expr::Call(call.clone())) { return self.emit_step(call, expr); } + // Check for SDK-level calls: step(), sleep(), waitForApproval() + if let Some(result) = self.try_emit_sdk_call(call, expr) { + return Some(result); + } } // await Promise.all([task_fn(...), ...]) if Self::is_promise_all(&await_expr.arg) { @@ -318,17 +341,70 @@ impl TsWacWalker { None } + /// Try to emit a node for SDK-level calls: step(), sleep(), waitForApproval() + fn try_emit_sdk_call(&mut self, call: &CallExpr, expr: &Expr) -> Option<(String, String)> { + let callee_name = match &call.callee { + Callee::Expr(callee) => match callee.as_ref() { + Expr::Ident(ident) => Some(ident.sym.as_ref().to_string()), + _ => None, + }, + _ => None, + }?; + + let line = self.span_line(expr.span()); + + match callee_name.as_str() { + "step" => { + // step("name", fn) — extract the name from the first string argument + let name = call + .args + .first() + .and_then(|a| extract_string_lit(&a.expr)) + .unwrap_or_else(|| "step".to_string()); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::InlineStep { name: name.clone() }, + label: name, + line, + }); + Some((node_id.clone(), node_id)) + } + "sleep" => { + // sleep(N) — extract the duration from the first argument + let seconds = call + .args + .first() + .map(|a| { + self.cm + .span_to_snippet(a.expr.span()) + .unwrap_or_else(|_| "?".to_string()) + }) + .unwrap_or_else(|| "?".to_string()); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::Sleep { seconds: seconds.clone() }, + label: format!("sleep({seconds})"), + line, + }); + Some((node_id.clone(), node_id)) + } + "waitForApproval" => { + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::WaitForApproval, + label: "waitForApproval".to_string(), + line, + }); + Some((node_id.clone(), node_id)) + } + _ => None, + } + } + fn emit_step(&mut self, call: &CallExpr, expr: &Expr) -> Option<(String, String)> { - if self.in_try { - self.errors - .push(validation::error_step_in_catch(self.span_line(expr.span()))); - return None; - } - if self.in_while { - self.errors - .push(validation::error_step_in_while(self.span_line(expr.span()))); - return None; - } if self.in_nested_func { self.errors.push(validation::error_step_in_nested_function( self.span_line(expr.span()), @@ -350,17 +426,6 @@ impl TsWacWalker { } fn emit_parallel(&mut self, promise_call: &CallExpr, expr: &Expr) -> Option<(String, String)> { - if self.in_try { - self.errors - .push(validation::error_step_in_catch(self.span_line(expr.span()))); - return None; - } - if self.in_while { - self.errors - .push(validation::error_step_in_while(self.span_line(expr.span()))); - return None; - } - let line = self.span_line(expr.span()); let start_id = self.next_id(); let start_node_id = self.add_node(DagNode { @@ -457,7 +522,16 @@ impl TsWacWalker { Some((branch_node_id, last_ids.into_iter().next().unwrap())) } else { let merge_id = format!("{branch_id}_merge"); - Some((branch_node_id, merge_id)) + let merge_node_id = self.add_node(DagNode { + id: merge_id, + node_type: DagNodeType::Merge, + label: "merge".to_string(), + line, + }); + for last in last_ids { + self.add_edge(&last, &merge_node_id, None); + } + Some((branch_node_id, merge_node_id)) } } @@ -473,7 +547,7 @@ impl TsWacWalker { return None; } let iter_source = self.expr_to_source(&for_in.right); - self.walk_loop_body_with_iter(&for_in.body, for_in.span, &iter_source) + self.walk_loop_body_with_iter(&for_in.body, for_in.span, &iter_source, "for") } fn walk_for_of(&mut self, for_of: &ForOfStmt) -> Option<(String, String)> { @@ -481,7 +555,7 @@ impl TsWacWalker { return None; } let iter_source = self.expr_to_source(&for_of.right); - self.walk_loop_body_with_iter(&for_of.body, for_of.span, &iter_source) + self.walk_loop_body_with_iter(&for_of.body, for_of.span, &iter_source, "for") } fn walk_loop_body( @@ -490,7 +564,7 @@ impl TsWacWalker { span: swc_common::Span, _label: &str, ) -> Option<(String, String)> { - self.walk_loop_body_with_iter(body, span, "...") + self.walk_loop_body_with_iter(body, span, "...", "for") } fn walk_loop_body_with_iter( @@ -498,13 +572,14 @@ impl TsWacWalker { body: &Stmt, span: swc_common::Span, iter_source: &str, + loop_label: &str, ) -> Option<(String, String)> { let line = self.span_line(span); let start_id = self.next_id(); let start_node_id = self.add_node(DagNode { id: start_id.clone(), node_type: DagNodeType::LoopStart { iter_source: iter_source.to_string() }, - label: "for".to_string(), + label: loop_label.to_string(), line, }); @@ -526,12 +601,11 @@ impl TsWacWalker { } fn walk_while(&mut self, while_stmt: &WhileStmt) -> Option<(String, String)> { - if self.stmt_contains_step(&while_stmt.body) { - self.errors.push(validation::error_step_in_while( - self.span_line(while_stmt.span), - )); + if !self.stmt_contains_step(&while_stmt.body) { + return None; } - None + let condition = self.expr_to_source(&while_stmt.test); + self.walk_loop_body_with_iter(&while_stmt.body, while_stmt.span, &condition, "while") } fn walk_try(&mut self, try_stmt: &TryStmt) -> Option<(String, String)> { @@ -545,12 +619,62 @@ impl TsWacWalker { .as_ref() .map_or(false, |f| self.body_contains_step(&f.stmts)); - if has_steps { - self.errors.push(validation::error_step_in_catch( - self.span_line(try_stmt.span), - )); + if !has_steps { + return None; } - None + + let line = self.span_line(try_stmt.span); + let branch_id = self.next_id(); + let branch_node_id = self.add_node(DagNode { + id: branch_id.clone(), + node_type: DagNodeType::Branch { condition_source: "try/catch".to_string() }, + label: "try".to_string(), + line, + }); + + let mut last_ids = Vec::new(); + + // Try body + if let Some((try_first, try_last)) = self.walk_body(&try_stmt.block.stmts) { + self.add_edge(&branch_node_id, &try_first, Some("try".to_string())); + last_ids.push(try_last); + } else { + last_ids.push(branch_node_id.clone()); + } + + // Catch body + if let Some(handler) = &try_stmt.handler { + if let Some((catch_first, catch_last)) = self.walk_body(&handler.body.stmts) { + self.add_edge(&branch_node_id, &catch_first, Some("catch".to_string())); + last_ids.push(catch_last); + } + } + + // Finally body — sequential after merge + let merge_last = if last_ids.len() == 1 { + last_ids.into_iter().next().unwrap() + } else { + let merge_id = format!("{branch_id}_merge"); + let merge_node_id = self.add_node(DagNode { + id: merge_id, + node_type: DagNodeType::Merge, + label: "merge".to_string(), + line, + }); + for last in last_ids { + self.add_edge(&last, &merge_node_id, None); + } + merge_node_id + }; + + if let Some(finalizer) = &try_stmt.finalizer { + if let Some((finally_first, finally_last)) = self.walk_body(&finalizer.stmts) { + self.add_edge(&merge_last, &finally_first, None); + return Some((branch_node_id, finally_last)); + } + } + + Some((branch_node_id, merge_last)) } fn walk_return(&mut self, ret: &ReturnStmt) -> Option<(String, String)> { @@ -632,6 +756,19 @@ pub fn parse_ts_workflow(code: &str) -> Result> { } } } + // export const main = workflow(async (...) => { ... }) + if let ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(export)) = item { + if let Decl::Var(var_decl) = &export.decl { + for decl in &var_decl.decls { + if let Some(init) = &decl.init { + if let Some(result) = find_workflow_call(init, &cm) { + workflow_body = Some(result); + break; + } + } + } + } + } } let (stmts, params) = workflow_body.ok_or_else(|| { diff --git a/backend/parsers/windmill-parser-wac/src/validation.rs b/backend/parsers/windmill-parser-wac/src/validation.rs index e3a57c6811..a76d2b1c17 100644 --- a/backend/parsers/windmill-parser-wac/src/validation.rs +++ b/backend/parsers/windmill-parser-wac/src/validation.rs @@ -12,23 +12,6 @@ impl std::fmt::Display for CompileError { } } -pub fn error_step_in_try(line: usize) -> CompileError { - CompileError { - message: - "Task calls inside try/except are not allowed. Steps have built-in error handling." - .to_string(), - line, - } -} - -pub fn error_step_in_while(line: usize) -> CompileError { - CompileError { - message: "Task calls inside while loops are not allowed. Use for loops instead." - .to_string(), - line, - } -} - pub fn error_step_in_nested_function(line: usize) -> CompileError { CompileError { message: "Task calls inside nested functions, closures, or lambdas are not allowed." @@ -53,12 +36,3 @@ pub fn error_missing_await(line: usize) -> CompileError { line, } } - -pub fn error_step_in_catch(line: usize) -> CompileError { - CompileError { - message: - "Task calls inside catch blocks are not allowed. Steps have built-in error handling." - .to_string(), - line, - } -} diff --git a/backend/parsers/windmill-parser-wac/tests/python_tests.rs b/backend/parsers/windmill-parser-wac/tests/python_tests.rs index 59f0b59f5c..cf117da9e7 100644 --- a/backend/parsers/windmill-parser-wac/tests/python_tests.rs +++ b/backend/parsers/windmill-parser-wac/tests/python_tests.rs @@ -147,7 +147,7 @@ async def my_etl(items: list): } #[test] -fn test_reject_step_in_try() { +fn test_step_in_try_except() { let code = r#" import asyncio from wmill import workflow, task @@ -155,39 +155,52 @@ from wmill import workflow, task @task async def extract_data(): ... +@task +async def handle_error(): ... + @workflow async def my_etl(): try: await extract_data() except Exception: - pass + await handle_error() "#; - let result = parse_python_workflow(code); - assert!(result.is_err()); - let errors = result.unwrap_err(); - assert!(errors[0].message.contains("try/except")); + let dag = parse_python_workflow(code).expect("should parse try/except"); + // Branch(try/except), extract_data, handle_error, merge = 4 + assert_eq!(dag.nodes.len(), 4); + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Branch { .. })); + assert_eq!(dag.nodes[0].label, "try"); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[2].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[3].node_type, DagNodeType::Merge)); } #[test] -fn test_reject_step_in_while() { +fn test_step_in_while() { let code = r#" import asyncio from wmill import workflow, task @task -async def extract_data(): ... +async def poll_status(): ... @workflow async def my_etl(): while True: - await extract_data() + await poll_status() "#; - let result = parse_python_workflow(code); - assert!(result.is_err()); - let errors = result.unwrap_err(); - assert!(errors[0].message.contains("while")); + let dag = parse_python_workflow(code).expect("should parse while loop"); + // LoopStart, poll_status, LoopEnd = 3 + assert_eq!(dag.nodes.len(), 3); + assert!(matches!( + dag.nodes[0].node_type, + DagNodeType::LoopStart { .. } + )); + assert_eq!(dag.nodes[0].label, "while"); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[2].node_type, DagNodeType::LoopEnd)); } #[test] @@ -264,3 +277,89 @@ async def my_wf(x: int): _ => panic!("expected Step node"), } } + +#[test] +fn test_task_script_and_task_flow_py() { + let code = r#" +from wmill import workflow, task, task_script, task_flow + +helper = task_script("./helper.py") +pipeline = task_flow("f/etl/pipeline") + +@task() +async def process(x: str) -> str: + return f"processed: {x}" + +@workflow +async def main(x: str): + a = await process(x=x) + b = await helper(a=a) + c = await pipeline(b=b) + return {"a": a, "b": b, "c": c} +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 4); // 3 steps + 1 return + + match &dag.nodes[1].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "helper"); + assert_eq!(script, "./helper.py"); + } + _ => panic!("expected Step node for task_script"), + } + + match &dag.nodes[2].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "pipeline"); + assert_eq!(script, "f/etl/pipeline"); + } + _ => panic!("expected Step node for task_flow"), + } +} + +#[test] +fn test_full_template_with_sdk_calls_py() { + let code = r#" +from wmill import workflow, task, task_script, step, sleep, wait_for_approval, get_resume_urls + +helper = task_script("./helper.py") + +@task() +async def process(x: str) -> str: + return f"processed: {x}" + +@workflow +async def main(x: str): + a = await process(x=x) + b = await helper(a=a) + urls = await step("get_urls", lambda: get_resume_urls()) + await sleep(1) + approval = await wait_for_approval(timeout=3600) + return {"processed": a, "helper_result": b, "approval": approval} +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + // process, helper, step("get_urls"), sleep(1), wait_for_approval, return = 6 + assert_eq!(dag.nodes.len(), 6); + + match &dag.nodes[2].node_type { + DagNodeType::InlineStep { name } => { + assert_eq!(name, "get_urls"); + } + _ => panic!("expected InlineStep node, got {:?}", dag.nodes[2].node_type), + } + + match &dag.nodes[3].node_type { + DagNodeType::Sleep { seconds } => { + assert_eq!(seconds, "1"); + } + _ => panic!("expected Sleep node, got {:?}", dag.nodes[3].node_type), + } + + assert!(matches!( + dag.nodes[4].node_type, + DagNodeType::WaitForApproval + )); + assert!(matches!(dag.nodes[5].node_type, DagNodeType::Return)); +} diff --git a/backend/parsers/windmill-parser-wac/tests/ts_tests.rs b/backend/parsers/windmill-parser-wac/tests/ts_tests.rs index 949f326b90..bb006c6744 100644 --- a/backend/parsers/windmill-parser-wac/tests/ts_tests.rs +++ b/backend/parsers/windmill-parser-wac/tests/ts_tests.rs @@ -129,45 +129,56 @@ export default workflow(async (items: string[]) => { } #[test] -fn test_reject_step_in_try_catch() { +fn test_step_in_try_catch() { let code = r#" import { workflow, task } from "windmill-client"; const extract_data = task(async () => {}); +const handle_error = task(async (e: any) => {}); export default workflow(async () => { try { await extract_data(); } catch (e) { - console.log(e); + await handle_error(e); } }); "#; - let result = parse_ts_workflow(code); - assert!(result.is_err()); - let errors = result.unwrap_err(); - assert!(errors[0].message.contains("catch")); + let dag = parse_ts_workflow(code).expect("should parse try/catch"); + // Branch(try/catch), extract_data, handle_error, merge = 4 + assert_eq!(dag.nodes.len(), 4); + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Branch { .. })); + assert_eq!(dag.nodes[0].label, "try"); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[2].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[3].node_type, DagNodeType::Merge)); } #[test] -fn test_reject_step_in_while_ts() { +fn test_step_in_while_ts() { let code = r#" import { workflow, task } from "windmill-client"; -const extract_data = task(async () => {}); +const poll_status = task(async () => {}); export default workflow(async () => { while (true) { - await extract_data(); + await poll_status(); } }); "#; - let result = parse_ts_workflow(code); - assert!(result.is_err()); - let errors = result.unwrap_err(); - assert!(errors[0].message.contains("while")); + let dag = parse_ts_workflow(code).expect("should parse while loop"); + // LoopStart, poll_status, LoopEnd = 3 + assert_eq!(dag.nodes.len(), 3); + assert!(matches!( + dag.nodes[0].node_type, + DagNodeType::LoopStart { .. } + )); + assert_eq!(dag.nodes[0].label, "while"); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[2].node_type, DagNodeType::LoopEnd)); } #[test] @@ -243,3 +254,158 @@ export default workflow(async (x: number) => { _ => panic!("expected Step node"), } } + +#[test] +fn test_task_script_and_task_flow() { + let code = r#" +import { workflow, task, taskScript, taskFlow } from "windmill-client"; + +const helper = taskScript("./helper.ts"); +const pipeline = taskFlow("f/etl/pipeline"); +const process = task(async (x: string) => {}); + +export default workflow(async (x: string) => { + const a = await process(x); + const b = await helper({ a }); + const c = await pipeline({ b }); + return { a, b, c }; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 4); // 3 steps + 1 return + + match &dag.nodes[0].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "process"); + assert_eq!(script, "process"); + } + _ => panic!("expected Step node"), + } + + match &dag.nodes[1].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "helper"); + assert_eq!(script, "./helper.ts"); + } + _ => panic!("expected Step node for taskScript"), + } + + match &dag.nodes[2].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "pipeline"); + assert_eq!(script, "f/etl/pipeline"); + } + _ => panic!("expected Step node for taskFlow"), + } +} + +#[test] +fn test_full_template_with_sdk_calls() { + let code = r#" +import { task, taskScript, step, sleep, waitForApproval, getResumeUrls, workflow } from "windmill-client"; + +const helper = taskScript("./helper.ts"); +const process = task(async (x: string): Promise => { + return `processed: ${x}`; +}); + +export const main = workflow(async (x: string) => { + const a = await process(x); + const b = await helper({ a }); + const urls = await step("get_urls", () => getResumeUrls()); + await sleep(1); + const approval = await waitForApproval({ timeout: 3600 }); + return { processed: a, helper_result: b, approval }; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + // process, helper, step("get_urls"), sleep(1), waitForApproval, return = 6 + assert_eq!(dag.nodes.len(), 6); + assert_eq!(dag.edges.len(), 5); + + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. })); + + match &dag.nodes[1].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "helper"); + assert_eq!(script, "./helper.ts"); + } + _ => panic!("expected Step node"), + } + + match &dag.nodes[2].node_type { + DagNodeType::InlineStep { name } => { + assert_eq!(name, "get_urls"); + } + _ => panic!("expected InlineStep node, got {:?}", dag.nodes[2].node_type), + } + + match &dag.nodes[3].node_type { + DagNodeType::Sleep { seconds } => { + assert_eq!(seconds, "1"); + } + _ => panic!("expected Sleep node, got {:?}", dag.nodes[3].node_type), + } + + assert!(matches!( + dag.nodes[4].node_type, + DagNodeType::WaitForApproval + )); + assert!(matches!(dag.nodes[5].node_type, DagNodeType::Return)); +} + +#[test] +fn test_complex_mixed_workflow() { + let code = r#" +import { workflow, task, step, sleep } from "windmill-client"; + +const validate = task(async (data: any) => {}); +const process_csv = task(async (data: any) => {}); +const process_json = task(async (data: any) => {}); +const enrich = task(async (item: any) => {}); +const store = task(async (data: any) => {}); + +export default workflow(async (data: any) => { + const validated = await validate(data); + if (validated.format === "csv") { + const parsed = await process_csv(validated); + for (const row of parsed.rows) { + await enrich(row); + } + } else { + await process_json(validated); + } + await sleep(5); + const ts = await step("timestamp", () => new Date().toISOString()); + await store(validated); + return { done: true }; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + + // validate, Branch, process_csv, LoopStart, enrich, LoopEnd, process_json, + // merge, sleep(5), step("timestamp"), store, return = 12 + assert_eq!(dag.nodes.len(), 12); + + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::Branch { .. })); + assert!(matches!(dag.nodes[2].node_type, DagNodeType::Step { .. })); // process_csv + assert!(matches!( + dag.nodes[3].node_type, + DagNodeType::LoopStart { .. } + )); + assert!(matches!(dag.nodes[4].node_type, DagNodeType::Step { .. })); // enrich + assert!(matches!(dag.nodes[5].node_type, DagNodeType::LoopEnd)); + assert!(matches!(dag.nodes[6].node_type, DagNodeType::Step { .. })); // process_json + assert!(matches!(dag.nodes[7].node_type, DagNodeType::Merge)); + assert!(matches!(dag.nodes[8].node_type, DagNodeType::Sleep { .. })); + assert!(matches!( + dag.nodes[9].node_type, + DagNodeType::InlineStep { .. } + )); // timestamp + assert!(matches!(dag.nodes[10].node_type, DagNodeType::Step { .. })); // store + assert!(matches!(dag.nodes[11].node_type, DagNodeType::Return)); +} diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 895a35713e..edc19c4a29 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -38,6 +38,7 @@ csharp-parser = [ "dep:windmill-parser-csharp"] nu-parser = [ "dep:windmill-parser-nu"] java-parser = [ "dep:windmill-parser-java"] ruby-parser = [ "dep:windmill-parser-ruby"] +r-parser = [ "dep:windmill-parser-r"] wac-parser = [ "dep:windmill-parser-wac"] asset-parser = [ "dep:windmill-parser-ts-asset", "dep:windmill-parser-py-asset", "dep:windmill-parser-sql-asset"] py-imports-parser = [ "dep:windmill-parser-py-imports"] @@ -58,6 +59,7 @@ windmill-parser-csharp = { workspace = true, optional = true } windmill-parser-nu = { workspace = true, optional = true } windmill-parser-java = { workspace = true, optional = true } windmill-parser-ruby = { workspace = true, optional = true } +windmill-parser-r = { workspace = true, optional = true } windmill-parser-wac = { workspace = true, optional = true } windmill-parser-ts-asset = { workspace = true, optional = true } windmill-parser-py-asset = { workspace = true, optional = true } diff --git a/backend/parsers/windmill-parser-wasm/build.nu b/backend/parsers/windmill-parser-wasm/build.nu index 212c5f3a37..2df321e374 100755 --- a/backend/parsers/windmill-parser-wasm/build.nu +++ b/backend/parsers/windmill-parser-wasm/build.nu @@ -55,6 +55,11 @@ const targets = [ desc: "Ruby", features: "ruby-parser", env: "tree-sitter", + }, { + ident: "r", + desc: "R", + features: "r-parser", + env: "tree-sitter", }, { ident: "wac", diff --git a/backend/parsers/windmill-parser-wasm/publish-pkgs.sh b/backend/parsers/windmill-parser-wasm/publish-pkgs.sh index 3ac0ceef18..42ae54f683 100755 --- a/backend/parsers/windmill-parser-wasm/publish-pkgs.sh +++ b/backend/parsers/windmill-parser-wasm/publish-pkgs.sh @@ -39,3 +39,6 @@ popd pushd "pkg-py-imports" && npm publish ${args} popd + +pushd "pkg-wac" && npm publish ${args} +popd diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index a2cef1b0ef..c843652e26 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -198,6 +198,12 @@ pub fn parse_ruby(code: &str) -> String { wrap_sig(windmill_parser_ruby::parse_ruby_signature(code)) } +#[cfg(feature = "r-parser")] +#[wasm_bindgen] +pub fn parse_r(code: &str) -> String { + wrap_sig(windmill_parser_r::parse_r_signature(code)) +} + #[cfg(feature = "asset-parser")] #[wasm_bindgen] pub fn parse_assets_sql(code: &str) -> String { diff --git a/backend/parsers/windmill-parser-wasm/wasm-sysroot/string.h b/backend/parsers/windmill-parser-wasm/wasm-sysroot/string.h index 30fce92495..4a2abf1142 100644 --- a/backend/parsers/windmill-parser-wasm/wasm-sysroot/string.h +++ b/backend/parsers/windmill-parser-wasm/wasm-sysroot/string.h @@ -1,5 +1,7 @@ #pragma once +#include + void *memcpy(void *dest, const void *src, unsigned long n); void *memmove(void *dest, const void *src, unsigned long n); void *memset(void *s, int c, unsigned long n); diff --git a/backend/src/main.rs b/backend/src/main.rs index 4fe22c517f..12a6616c51 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -51,10 +51,11 @@ use windmill_common::{ MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, - POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, - RUBY_REPOS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING, - TIMEOUT_WAIT_RESULT_SETTING, UV_INDEX_STRATEGY_SETTING, WORKSPACE_REGISTRIES_SETTING, + POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING, + RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, + SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, + UV_INDEX_STRATEGY_SETTING, WORKSPACE_REGISTRIES_SETTING, }, scripts::ScriptLang, stats_oss::schedule_stats, @@ -67,7 +68,7 @@ use windmill_common::{ is_native_mode_from_env, reload_custom_tags_setting, Connection, HUB_CACHE_DIR, HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, WINDMILL_DIR, WORKER_GROUP, }, - KillpillSender, DEFAULT_HUB_BASE_URL, METRICS_ENABLED, + KillpillSender, DEFAULT_HUB_BASE_URL, INSTANCE_NAME, METRICS_ENABLED, }; #[cfg(feature = "enterprise")] @@ -94,20 +95,21 @@ use windmill_worker::{ BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, JAVA_CACHE_DIR, NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, - RUBY_CACHE_DIR, RUST_CACHE_DIR, TAR_JAVA_CACHE_DIR, UV_CACHE_DIR, + RUBY_CACHE_DIR, RUST_CACHE_DIR, R_CACHE_DIR, TAR_JAVA_CACHE_DIR, UV_CACHE_DIR, }; use crate::monitor::{ - initial_load, load_keep_job_dir, load_metrics_debug_enabled, load_require_preexisting_user, - load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, monitor_db, - reload_app_workspaced_route_setting, reload_audit_log_retention_days_setting, - reload_base_url_setting, reload_bunfig_install_scopes_setting, - reload_critical_alert_mute_ui_setting, reload_critical_alerts_on_token_expiry_setting, - reload_critical_error_channels_setting, reload_extra_pip_index_url_setting, - reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting, - reload_hub_base_url_setting, reload_instance_events_webhook_setting, - reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, - reload_license_key, reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting, + initial_load, load_keep_job_dir, load_metrics_debug_enabled, load_preview_tags_override, + load_require_preexisting_user, load_tag_per_workspace_enabled, + load_tag_per_workspace_workspaces, monitor_db, reload_app_workspaced_route_setting, + reload_audit_log_retention_days_setting, reload_base_url_setting, + reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting, + reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting, + reload_extra_pip_index_url_setting, reload_http_route_workspaced_route_setting, + reload_hub_api_secret_setting, reload_hub_base_url_setting, + reload_instance_events_webhook_setting, reload_job_default_timeout_setting, + reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, + reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config, reload_uv_index_strategy_setting, reload_worker_config, MonitorIteration, }; @@ -1741,6 +1743,11 @@ async fn process_notify_event( ); } } + PREVIEW_TAGS_OVERRIDE_SETTING => { + if let Err(e) = load_preview_tags_override(db).await { + tracing::error!("Error loading preview tags override: {e:#}"); + } + } SMTP_SETTING => { reload_smtp_config(db).await; } @@ -1791,7 +1798,8 @@ async fn process_notify_event( reload_otel_tracing_proxy_setting(conn).await; if worker_mode { tracing::info!("OTEL tracing proxy setting changed, restarting worker"); - send_delayed_killpill(tx, 4, "OTEL tracing proxy setting change").await; + spawn_graceful_killpill(tx, db, 10, "OTEL tracing proxy setting change") + .await; } } REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => { @@ -1799,12 +1807,12 @@ async fn process_notify_event( } EXPOSE_METRICS_SETTING => { tracing::info!("Metrics setting changed, restarting"); - send_delayed_killpill(tx, 40, "metrics setting change").await; + spawn_graceful_killpill(tx, db, 10, "metrics setting change").await; } EMAIL_DOMAIN_SETTING => { tracing::info!("Email domain setting changed"); if server_mode { - send_delayed_killpill(tx, 4, "email domain setting change").await; + spawn_graceful_killpill(tx, db, 10, "email domain setting change").await; } } EXPOSE_DEBUG_METRICS_SETTING => { @@ -1840,19 +1848,19 @@ async fn process_notify_event( } OTEL_SETTING => { tracing::info!("OTEL setting changed, restarting"); - send_delayed_killpill(tx, 4, "OTEL setting change").await; + spawn_graceful_killpill(tx, db, 10, "OTEL setting change").await; } REQUEST_SIZE_LIMIT_SETTING => { if server_mode { tracing::info!("Request limit size change detected, killing server expecting to be restarted"); - send_delayed_killpill(tx, 4, "request size limit change").await; + spawn_graceful_killpill(tx, db, 10, "request size limit change").await; } } SAML_METADATA_SETTING => { tracing::info!( "SAML metadata change detected, killing server expecting to be restarted" ); - send_delayed_killpill(tx, 0, "SAML metadata change").await; + spawn_graceful_killpill(tx, db, 10, "SAML metadata change").await; } HUB_BASE_URL_SETTING => { if let Err(e) = reload_hub_base_url_setting(conn, server_mode).await { @@ -1901,6 +1909,24 @@ async fn process_notify_event( .unwrap_or(false); tracing::info!("Workspace telemetry setting changed: enabled={}", enabled); } + RESTART_COORDINATION_SETTING => { + // Internal coordination key for staggered restarts, no action needed + } + "plain_emails_telemetry" => { + let enabled = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = 'plain_emails_telemetry'" + ) + .fetch_optional(db) + .await + .ok() + .flatten() + .and_then(|v| v.as_bool()) + .unwrap_or(false); + tracing::info!( + "Plain emails telemetry setting changed: enabled={}", + enabled + ); + } _ => { tracing::info!("Unrecognized Global Setting Change Payload: {:?}", payload); } @@ -1985,6 +2011,7 @@ pub async fn run_workers( &*POWERSHELL_CACHE_DIR, &*JAVA_CACHE_DIR, &*RUBY_CACHE_DIR, + &*R_CACHE_DIR, &*TAR_JAVA_CACHE_DIR, // for related places search: ADD_NEW_LANG ] { DirBuilder::new() @@ -2042,14 +2069,145 @@ pub async fn run_workers( Ok(()) } -async fn send_delayed_killpill(tx: &KillpillSender, mut max_delay_secs: u64, context: &str) { - if max_delay_secs == 0 { - max_delay_secs = 1; - } - // Random delay to avoid all servers/workers shutting down simultaneously - let rd_delay = rand::rng().random_range(0..max_delay_secs); - tracing::info!("Scheduling {context} shutdown in {rd_delay}s"); - tokio::time::sleep(Duration::from_secs(rd_delay)).await; +/// Schedule a graceful restart with DB-coordinated staggering. +/// +/// Uses a PostgreSQL advisory lock to serialize restart scheduling across server instances. +/// Each instance records its planned restart time in the `_restart_coordination` global setting; +/// subsequent instances read existing schedules and shift their restart to maintain at least +/// `safety_margin_secs` between consecutive restarts (must exceed the server startup time). +/// +/// Every server waits at least `DRAIN_DELAY_SECS` to let in-flight requests complete. +/// Each subsequent server waits an additional `safety_margin_secs` after the previous one, +/// guaranteeing zero downtime overlap. +/// +/// The DB coordination is done synchronously (fast, ~ms) to reserve our restart slot, +/// then the sleep+kill is spawned in the background so the notification handler is not blocked. +/// +/// Falls back to drain-only delay if DB coordination fails. +async fn spawn_graceful_killpill( + tx: &KillpillSender, + db: &Pool, + safety_margin_secs: u64, + context: &str, +) { + // Minimum delay before any restart to let in-flight requests drain + const DRAIN_DELAY_SECS: u64 = 3; - tx.send(); + let delay = match coordinate_restart_delay(db, safety_margin_secs, DRAIN_DELAY_SECS).await { + Ok(d) => d, + Err(e) => { + tracing::warn!( + "Failed to coordinate restart for {context}: {e:#}, \ + falling back to drain delay of {DRAIN_DELAY_SECS}s" + ); + DRAIN_DELAY_SECS + } + }; + + tracing::info!("Scheduling {context} graceful shutdown in {delay}s"); + let tx = tx.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(delay)).await; + tx.send(); + }); +} + +/// Coordinate a restart delay with other instances via the DB. +/// +/// Returns the delay (in seconds from now) at which this instance should restart. +/// The first server gets `drain_delay_secs` (to let in-flight requests complete). +/// Each subsequent server is spaced `safety_margin_secs` after the latest scheduled restart. +async fn coordinate_restart_delay( + db: &Pool, + safety_margin_secs: u64, + drain_delay_secs: u64, +) -> anyhow::Result { + const RESTART_LOCK_ID: i64 = 737_483_920; + // Stale threshold: ignore coordination entries older than this + const STALE_THRESHOLD_SECS: i64 = 120; + + let now = chrono::Utc::now(); + + let mut tx = db.begin().await.context("begin restart coordination tx")?; + + // Serialize access across all instances + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(RESTART_LOCK_ID) + .execute(&mut *tx) + .await + .context("acquire restart coordination lock")?; + + // Read existing coordination record + let existing: Option = + sqlx::query_scalar("SELECT value FROM global_settings WHERE name = $1") + .bind(RESTART_COORDINATION_SETTING) + .fetch_optional(&mut *tx) + .await + .context("read restart coordination")?; + + // Parse existing scheduled restarts, filtering out stale entries + // Each entry is (instance_name, restart_at) + let mut scheduled: Vec<(String, chrono::DateTime)> = Vec::new(); + if let Some(val) = &existing { + if let Some(arr) = val.get("restarts").and_then(|v| v.as_array()) { + for entry in arr { + let instance = entry + .get("instance") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_string(); + if let Some(ts_str) = entry.get("restart_at").and_then(|v| v.as_str()) { + if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(ts_str) { + let dt = dt.with_timezone(&chrono::Utc); + let stale_cutoff = now - chrono::Duration::seconds(STALE_THRESHOLD_SECS); + if dt > stale_cutoff { + scheduled.push((instance, dt)); + } + } + } + } + } + } + + // Find the latest scheduled restart + let latest = scheduled.iter().map(|(_, dt)| *dt).max(); + let earliest_allowed = now + chrono::Duration::seconds(drain_delay_secs as i64); + + // Our restart time: drain_delay from now, or safety_margin after the latest existing restart + let our_restart = match latest { + Some(last) => { + let after_last = last + chrono::Duration::seconds(safety_margin_secs as i64); + // Use whichever is later: drain delay or staggered position + earliest_allowed.max(after_last) + } + None => earliest_allowed, + }; + + // Record our restart time (deduplicate: remove any prior entry for this instance) + scheduled.retain(|(inst, _)| inst != &*INSTANCE_NAME); + scheduled.push((INSTANCE_NAME.clone(), our_restart)); + let new_value = serde_json::json!({ + "restarts": scheduled.iter().map(|(inst, dt)| { + serde_json::json!({ + "instance": inst, + "restart_at": dt.to_rfc3339() + }) + }).collect::>() + }); + + sqlx::query( + "INSERT INTO global_settings (name, value, updated_at) \ + VALUES ($1, $2, now()) \ + ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()", + ) + .bind(RESTART_COORDINATION_SETTING) + .bind(&new_value) + .execute(&mut *tx) + .await + .context("write restart coordination")?; + + tx.commit().await.context("commit restart coordination")?; + + let delay = (our_restart - now).num_seconds().max(0) as u64; + Ok(delay) } diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index bb7640c5ed..5e87f3d161 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -62,7 +62,7 @@ use windmill_common::{ KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, - POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_INDEX_STRATEGY_SETTING, @@ -79,8 +79,8 @@ use windmill_common::{ load_periodic_bash_script_interval_from_env, load_whitelist_env_vars_from_env, load_worker_config, reload_custom_tags_setting, store_pull_query, store_suspended_pull_query, Connection, WorkerConfig, DEFAULT_TAGS_PER_WORKSPACE, - DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, WINDMILL_DIR, - WORKER_CONFIG, WORKER_GROUP, + DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, PREVIEW_TAGS_OVERRIDE, SCRIPT_TOKEN_EXPIRY, + SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG, WORKER_GROUP, }, KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, @@ -169,6 +169,8 @@ lazy_static::lazy_static! { static ref QUEUE_COUNT_TAGS: Arc>> = Arc::new(RwLock::new(Vec::new())); static ref QUEUE_RUNNING_COUNT_TAGS: Arc>> = Arc::new(RwLock::new(Vec::new())); + static ref OTEL_QUEUE_COUNT_TAGS: Arc>> = Arc::new(RwLock::new(Vec::new())); + static ref OTEL_QUEUE_RUNNING_COUNT_TAGS: Arc>> = Arc::new(RwLock::new(Vec::new())); static ref DISABLE_CONCURRENCY_LIMIT: bool = std::env::var("DISABLE_CONCURRENCY_LIMIT").is_ok_and(|s| s == "true"); //legacy typo @@ -233,6 +235,10 @@ pub async fn initial_load( if let Err(e) = load_tag_per_workspace_workspaces(db).await { tracing::error!("Error loading default tag per workpsace workspaces: {e:#}"); } + + if let Err(e) = load_preview_tags_override(db).await { + tracing::error!("Error loading preview tags override: {e:#}"); + } } if server_mode { @@ -497,6 +503,16 @@ pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> { Ok(()) } +pub async fn load_preview_tags_override(db: &DB) -> error::Result<()> { + let value = load_value_from_global_settings(db, PREVIEW_TAGS_OVERRIDE_SETTING).await; + + match value { + Ok(Some(serde_json::Value::Bool(t))) => PREVIEW_TAGS_OVERRIDE.store(t, Ordering::Relaxed), + _ => (), + }; + Ok(()) +} + pub async fn reload_critical_alert_mute_ui_setting(conn: &Connection) -> error::Result<()> { if let Ok(Some(serde_json::Value::Bool(t))) = load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true).await @@ -2372,8 +2388,20 @@ pub async fn expose_queue_metrics(db: &Pool) { } } + let otel_enabled = OTEL_METRICS_ENABLED.load(Ordering::Relaxed); + + if otel_enabled { + for q in OTEL_QUEUE_COUNT_TAGS.read().await.iter() { + if queue_counts.get(q).is_none() { + otel_set_queue_count(q, 0); + } + } + } + #[allow(unused_mut)] let mut tags_to_watch = vec![]; + #[allow(unused_mut)] + let mut otel_tags_to_watch = vec![]; for q in queue_counts { let count = q.1; let tag = q.0; @@ -2385,6 +2413,9 @@ pub async fn expose_queue_metrics(db: &Pool) { tags_to_watch.push(tag.to_string()); } + if otel_enabled { + otel_tags_to_watch.push(tag.to_string()); + } otel_set_queue_count(&tag, count as i64); // save queue_count and delay metrics per tag @@ -2419,9 +2450,13 @@ pub async fn expose_queue_metrics(db: &Pool) { let mut w = QUEUE_COUNT_TAGS.write().await; *w = tags_to_watch; } + if otel_enabled { + let mut w = OTEL_QUEUE_COUNT_TAGS.write().await; + *w = otel_tags_to_watch; + } // Single DB query for running counts, shared by Prometheus and OTel - let otel_running = OTEL_METRICS_ENABLED.load(Ordering::Relaxed); + let otel_running = otel_enabled; #[cfg(feature = "prometheus")] let need_running_counts = metrics_enabled || otel_running; #[cfg(not(feature = "prometheus"))] @@ -2439,8 +2474,18 @@ pub async fn expose_queue_metrics(db: &Pool) { } } + if otel_running { + for q in OTEL_QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { + if queue_running_counts.get(q).is_none() { + otel_set_queue_running_count(q, 0); + } + } + } + #[allow(unused_mut, unused_variables)] let mut running_tags_to_watch: Vec = vec![]; + #[allow(unused_mut, unused_variables)] + let mut otel_running_tags_to_watch: Vec = vec![]; for (tag, count) in &queue_running_counts { #[cfg(feature = "prometheus")] if metrics_enabled { @@ -2451,6 +2496,7 @@ pub async fn expose_queue_metrics(db: &Pool) { if otel_running { otel_set_queue_running_count(tag, *count as i64); + otel_running_tags_to_watch.push(tag.to_string()); } } @@ -2459,6 +2505,10 @@ pub async fn expose_queue_metrics(db: &Pool) { let mut w = QUEUE_RUNNING_COUNT_TAGS.write().await; *w = running_tags_to_watch; } + if otel_running { + let mut w = OTEL_QUEUE_RUNNING_COUNT_TAGS.write().await; + *w = otel_running_tags_to_watch; + } } } diff --git a/backend/substitute_ee_code.sh b/backend/substitute_ee_code.sh index 63d39e3854..cfe397263d 100755 --- a/backend/substitute_ee_code.sh +++ b/backend/substitute_ee_code.sh @@ -84,7 +84,7 @@ fi if [ "$REVERT" == "YES" ]; then backend_dirpath="${root_dirpath}/backend/" - for ce_file in $(find "${root_dirpath}/backend" -name "*_ee.rs"); do + for ce_file in $(find "${root_dirpath}/backend" \( -name "*_ee.rs" -o -name "ee.rs" \)); do if [ -L "${ce_file}" ]; then rm "${ce_file}" echo "Deleted symlink '${ce_file}'" diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 481ff43ea7..582bf7b684 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -26,7 +26,7 @@ native_trigger_service: nextcloud request_type: sync, async, sync_sse runnable_type: ScriptHash, ScriptPath, FlowPath script_kind: script, trigger, failure, command, approval, preprocessor -script_lang: python3, deno, go, bash, postgresql, nativets, bun, mysql, bigquery, snowflake, graphql, powershell, mssql, php, bunnative, rust, ansible, csharp, oracledb, nu, java, duckdb, ruby +script_lang: python3, deno, go, bash, postgresql, nativets, bun, mysql, bigquery, snowflake, graphql, powershell, mssql, php, bunnative, rust, ansible, csharp, oracledb, nu, java, duckdb, ruby, rlang trigger_kind: webhook, http, websocket, kafka, email, nats, postgres, sqs, mqtt, gcp, default_email, nextcloud trigger_mode: enabled, disabled, suspended workspace_key_kind: cloud diff --git a/backend/tests/otel.rs b/backend/tests/otel.rs new file mode 100644 index 0000000000..dd56cbf425 --- /dev/null +++ b/backend/tests/otel.rs @@ -0,0 +1,504 @@ +//! E2E tests for OpenTelemetry integration. +//! +//! Verify that metrics are recorded with correct names/values/attributes and +//! spans are created with correct trace IDs, attributes, and status codes. +//! +//! Run with: cargo test --features enterprise,private,otel --test otel -- --test-threads=1 + +#![cfg(all(feature = "otel", feature = "enterprise"))] + +use std::sync::{atomic::Ordering, Arc}; + +use opentelemetry::global; +use opentelemetry::trace::TracerProvider as _; +use opentelemetry_sdk::{ + metrics::{InMemoryMetricExporter, PeriodicReader, SdkMeterProvider}, + trace::{InMemorySpanExporter, SdkTracerProvider, SimpleSpanProcessor}, +}; +use windmill_common::otel_ee::*; +use windmill_common::{OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED}; + +// ── Global test infrastructure ────────────────────────────────────────── + +struct OtelTestState { + metric_exporter: InMemoryMetricExporter, + span_exporter: InMemorySpanExporter, + meter_provider: SdkMeterProvider, +} + +static STATE: tokio::sync::OnceCell> = tokio::sync::OnceCell::const_new(); + +async fn ensure_setup() -> Arc { + STATE + .get_or_init(|| async { + // Metrics: InMemoryMetricExporter + PeriodicReader (needs async tokio context) + let metric_exporter = InMemoryMetricExporter::default(); + let reader = PeriodicReader::builder(metric_exporter.clone()).build(); + let meter_provider = SdkMeterProvider::builder().with_reader(reader).build(); + global::set_meter_provider(meter_provider.clone()); + OTEL_METRICS_ENABLED.store(true, Ordering::SeqCst); + + // Tracing: InMemorySpanExporter + SimpleSpanProcessor + let span_exporter = InMemorySpanExporter::default(); + let tracer_provider = SdkTracerProvider::builder() + .with_span_processor(SimpleSpanProcessor::new(span_exporter.clone())) + .build(); + let tracer = tracer_provider.tracer("windmill"); + *TRACER.write().unwrap() = Some(tracer); + OTEL_TRACING_ENABLED.store(true, Ordering::SeqCst); + + Arc::new(OtelTestState { metric_exporter, span_exporter, meter_provider }) + }) + .await + .clone() +} + +// ── Metric helper: flush + collect ────────────────────────────────────── + +fn flush_and_get_metrics( + state: &OtelTestState, +) -> Vec { + state.meter_provider.force_flush().expect("flush failed"); + state + .metric_exporter + .get_finished_metrics() + .expect("get_finished_metrics failed") +} + +fn find_metric<'a>( + all: &'a [opentelemetry_sdk::metrics::data::ResourceMetrics], + name: &str, +) -> Option<&'a opentelemetry_sdk::metrics::data::Metric> { + all.iter() + .flat_map(|rm| rm.scope_metrics()) + .flat_map(|sm| sm.metrics()) + .find(|m| m.name() == name) +} + +fn metric_names(all: &[opentelemetry_sdk::metrics::data::ResourceMetrics]) -> Vec { + all.iter() + .flat_map(|rm| rm.scope_metrics()) + .flat_map(|sm| sm.metrics()) + .map(|m| m.name().to_string()) + .collect() +} + +// ── Counter value helpers ─────────────────────────────────────────────── + +fn sum_u64_value(metric: &opentelemetry_sdk::metrics::data::Metric) -> Option { + use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData}; + match metric.data() { + AggregatedMetrics::U64(MetricData::Sum(sum)) => { + Some(sum.data_points().map(|dp| dp.value()).sum()) + } + _ => None, + } +} + +fn gauge_i64_values( + metric: &opentelemetry_sdk::metrics::data::Metric, +) -> Vec<(Vec, i64)> { + use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData}; + match metric.data() { + AggregatedMetrics::I64(MetricData::Gauge(gauge)) => gauge + .data_points() + .map(|dp| (dp.attributes().cloned().collect(), dp.value())) + .collect(), + _ => panic!("expected I64 Gauge metric"), + } +} + +fn gauge_f64_value(metric: &opentelemetry_sdk::metrics::data::Metric) -> Option { + use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData}; + match metric.data() { + AggregatedMetrics::F64(MetricData::Gauge(gauge)) => { + gauge.data_points().next().map(|dp| dp.value()) + } + _ => None, + } +} + +fn histogram_f64_count(metric: &opentelemetry_sdk::metrics::data::Metric) -> Option { + use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData}; + match metric.data() { + AggregatedMetrics::F64(MetricData::Histogram(hist)) => { + Some(hist.data_points().map(|dp| dp.count()).sum()) + } + _ => None, + } +} + +fn histogram_f64_sum(metric: &opentelemetry_sdk::metrics::data::Metric) -> Option { + use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData}; + match metric.data() { + AggregatedMetrics::F64(MetricData::Histogram(hist)) => { + Some(hist.data_points().map(|dp| dp.sum()).sum()) + } + _ => None, + } +} + +// ═══════════════════════════════════════════════════════════════════════ +// METRICS E2E TEST +// +// All metric assertions live in one test function because the PeriodicReader's +// background task is tied to the tokio runtime that created it. Separate +// #[tokio::test] functions each get their own runtime, and the reader becomes +// disconnected after the first test's runtime is dropped. +// ═══════════════════════════════════════════════════════════════════════ + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_all_metrics_e2e() { + let state = ensure_setup().await; + + // ── Counters ──────────────────────────────────────────────────── + + otel_incr_queue_push_count(); + otel_incr_queue_push_count(); + otel_incr_queue_push_count(); + otel_incr_queue_delete_count(); + otel_incr_queue_pull_count(); + otel_incr_zombie_restart_count(7); + otel_incr_zombie_delete_count(3); + otel_incr_worker_execution_count("bun"); + otel_incr_worker_execution_count("bun"); + otel_incr_worker_execution_failed("go"); + otel_incr_worker_started(); + + // ── Gauges ────────────────────────────────────────────────────── + + otel_set_queue_count("python3", 42); + otel_set_queue_running_count("deno", 5); + otel_set_worker_busy("worker-test-1", 1); + otel_set_db_pool(5, 10, 20); + otel_set_health_db_latency(2.5); + otel_set_worker_uptime("w-uptime", 3600.0); + otel_set_health_status_phase("healthy"); + otel_set_health_db_unresponsive(true); + + // ── Histograms ────────────────────────────────────────────────── + + otel_record_worker_execution_duration("python3", 1.5); + otel_record_worker_execution_duration("python3", 2.5); + otel_record_worker_pull_duration("w1", true, 0.05); + otel_record_worker_pull_duration("w1", false, 0.01); + + // ── Flush and collect ─────────────────────────────────────────── + + let metrics = flush_and_get_metrics(&state); + let names = metric_names(&metrics); + + // ── Verify all 20 metric names are present ────────────────────── + + let expected = [ + "windmill.queue.push_count", + "windmill.queue.delete_count", + "windmill.queue.pull_count", + "windmill.queue.zombie_restart_count", + "windmill.queue.zombie_delete_count", + "windmill.queue.count", + "windmill.queue.running_count", + "windmill.worker.execution_count", + "windmill.worker.execution_duration", + "windmill.worker.busy", + "windmill.worker.pull_duration", + "windmill.worker.execution_failed", + "windmill.db.pool.active", + "windmill.db.pool.idle", + "windmill.db.pool.max", + "windmill.health.db_latency", + "windmill.worker.started", + "windmill.worker.uptime", + "windmill.health.status", + "windmill.health.db_unresponsive", + ]; + for name in expected { + assert!( + names.iter().any(|n| n == name), + "metric '{}' not found in {:?}", + name, + names + ); + } + + // ── Counter values ────────────────────────────────────────────── + + let m = find_metric(&metrics, "windmill.queue.push_count").unwrap(); + assert!(sum_u64_value(m).unwrap() >= 3, "push_count should be >= 3"); + + let m = find_metric(&metrics, "windmill.queue.delete_count").unwrap(); + assert!(sum_u64_value(m).unwrap() >= 1); + + let m = find_metric(&metrics, "windmill.queue.pull_count").unwrap(); + assert!(sum_u64_value(m).unwrap() >= 1); + + let m = find_metric(&metrics, "windmill.queue.zombie_restart_count").unwrap(); + assert!(sum_u64_value(m).unwrap() >= 7); + + let m = find_metric(&metrics, "windmill.queue.zombie_delete_count").unwrap(); + assert!(sum_u64_value(m).unwrap() >= 3); + + let m = find_metric(&metrics, "windmill.worker.execution_count").unwrap(); + assert!(sum_u64_value(m).unwrap() >= 2); + + let m = find_metric(&metrics, "windmill.worker.execution_failed").unwrap(); + assert!(sum_u64_value(m).unwrap() >= 1); + + let m = find_metric(&metrics, "windmill.worker.started").unwrap(); + assert!(sum_u64_value(m).unwrap() >= 1); + + // ── Gauge values ──────────────────────────────────────────────── + + let m = find_metric(&metrics, "windmill.queue.count").unwrap(); + let values = gauge_i64_values(m); + let dp = values + .iter() + .find(|(attrs, _)| { + attrs + .iter() + .any(|kv| kv.key.as_str() == "tag" && kv.value.as_str() == "python3") + }) + .expect("queue.count data point with tag=python3 not found"); + assert_eq!(dp.1, 42); + + let m = find_metric(&metrics, "windmill.queue.running_count").unwrap(); + let values = gauge_i64_values(m); + let dp = values + .iter() + .find(|(attrs, _)| { + attrs + .iter() + .any(|kv| kv.key.as_str() == "tag" && kv.value.as_str() == "deno") + }) + .expect("running_count data point with tag=deno not found"); + assert_eq!(dp.1, 5); + + let m = find_metric(&metrics, "windmill.worker.busy").unwrap(); + let values = gauge_i64_values(m); + let dp = values + .iter() + .find(|(attrs, _)| { + attrs + .iter() + .any(|kv| kv.key.as_str() == "worker" && kv.value.as_str() == "worker-test-1") + }) + .expect("worker.busy data point with worker=worker-test-1 not found"); + assert_eq!(dp.1, 1); + + let m = find_metric(&metrics, "windmill.db.pool.active").unwrap(); + assert_eq!(gauge_i64_values(m)[0].1, 5); + let m = find_metric(&metrics, "windmill.db.pool.idle").unwrap(); + assert_eq!(gauge_i64_values(m)[0].1, 10); + let m = find_metric(&metrics, "windmill.db.pool.max").unwrap(); + assert_eq!(gauge_i64_values(m)[0].1, 20); + + let m = find_metric(&metrics, "windmill.health.db_latency").unwrap(); + assert!((gauge_f64_value(m).unwrap() - 2.5).abs() < f64::EPSILON); + + let m = find_metric(&metrics, "windmill.worker.uptime").unwrap(); + assert!((gauge_f64_value(m).unwrap() - 3600.0).abs() < f64::EPSILON); + + let m = find_metric(&metrics, "windmill.health.db_unresponsive").unwrap(); + assert_eq!(gauge_i64_values(m)[0].1, 1); + + // ── Health status phase (all 3 phases) ────────────────────────── + + let m = find_metric(&metrics, "windmill.health.status").unwrap(); + let values = gauge_i64_values(m); + let healthy = values + .iter() + .find(|(attrs, _)| { + attrs + .iter() + .any(|kv| kv.key.as_str() == "phase" && kv.value.as_str() == "healthy") + }) + .expect("phase=healthy"); + let degraded = values + .iter() + .find(|(attrs, _)| { + attrs + .iter() + .any(|kv| kv.key.as_str() == "phase" && kv.value.as_str() == "degraded") + }) + .expect("phase=degraded"); + let unhealthy = values + .iter() + .find(|(attrs, _)| { + attrs + .iter() + .any(|kv| kv.key.as_str() == "phase" && kv.value.as_str() == "unhealthy") + }) + .expect("phase=unhealthy"); + assert_eq!(healthy.1, 1); + assert_eq!(degraded.1, 0); + assert_eq!(unhealthy.1, 0); + + // ── Histogram values ──────────────────────────────────────────── + + let m = find_metric(&metrics, "windmill.worker.execution_duration").unwrap(); + assert!(histogram_f64_count(m).unwrap() >= 2); + assert!(histogram_f64_sum(m).unwrap() >= 4.0); + + let m = find_metric(&metrics, "windmill.worker.pull_duration").unwrap(); + assert!(histogram_f64_count(m).unwrap() >= 2); +} + +// ═══════════════════════════════════════════════════════════════════════ +// SPAN E2E TESTS +// ═══════════════════════════════════════════════════════════════════════ + +fn make_test_job(id: uuid::Uuid, parent: Option) -> windmill_queue::MiniPulledJob { + use windmill_types::jobs::JobKind; + let mut job = windmill_queue::MiniPulledJob::new_inline( + "test-workspace".to_string(), + None, + "test-user".to_string(), + "u/test-user".to_string(), + "test@example.com".to_string(), + Some("f/test/script".to_string()), + JobKind::Script, + None, + "deno".to_string(), + None, + ); + job.id = id; + job.parent_job = parent; + job.started_at = Some(chrono::Utc::now()); + job +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_root_job_span_created_on_success() { + let state = ensure_setup().await; + state.span_exporter.reset(); + + let job_id = uuid::Uuid::new_v4(); + let job = make_test_job(job_id, None); + windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true); + + let spans = state.span_exporter.get_finished_spans().unwrap(); + let span = spans + .iter() + .find(|s| s.name == "full_job") + .expect("full_job span not found"); + + assert_eq!(span.status, opentelemetry::trace::Status::Ok,); + + // Verify attributes + let attrs: Vec<_> = span.attributes.iter().map(|kv| kv.key.as_str()).collect(); + assert!(attrs.contains(&"job_id"), "missing job_id attribute"); + assert!( + attrs.contains(&"workspace_id"), + "missing workspace_id attribute" + ); + assert!( + attrs.contains(&"script_path"), + "missing script_path attribute" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_root_job_span_error_on_failure() { + let state = ensure_setup().await; + state.span_exporter.reset(); + + let job_id = uuid::Uuid::new_v4(); + let job = make_test_job(job_id, None); + windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, false); + + let spans = state.span_exporter.get_finished_spans().unwrap(); + let span = spans + .iter() + .find(|s| s.name == "full_job") + .expect("full_job span not found"); + + match &span.status { + opentelemetry::trace::Status::Error { description } => { + assert_eq!(description.as_ref(), "Job failed"); + } + other => panic!("expected Error status, got {:?}", other), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_root_job_trace_id_matches_uuid() { + let state = ensure_setup().await; + state.span_exporter.reset(); + + let job_id = uuid::Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + let job = make_test_job(job_id, None); + windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true); + + let spans = state.span_exporter.get_finished_spans().unwrap(); + let span = spans + .iter() + .find(|s| s.name == "full_job") + .expect("full_job span not found"); + + let expected_trace_id = + opentelemetry::trace::TraceId::from_bytes(job_id.as_u128().to_be_bytes()); + assert_eq!(span.span_context.trace_id(), expected_trace_id); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_root_job_span_id_matches_uuid() { + let state = ensure_setup().await; + state.span_exporter.reset(); + + let job_id = uuid::Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + let job = make_test_job(job_id, None); + windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true); + + let spans = state.span_exporter.get_finished_spans().unwrap(); + let span = spans + .iter() + .find(|s| s.name == "full_job") + .expect("full_job span not found"); + + let expected_span_id = + opentelemetry::trace::SpanId::from_bytes(job_id.as_u64_pair().1.to_be_bytes()); + assert_eq!(span.span_context.span_id(), expected_span_id); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_child_job_produces_no_span() { + let state = ensure_setup().await; + state.span_exporter.reset(); + + let parent_id = uuid::Uuid::new_v4(); + let job_id = uuid::Uuid::new_v4(); + let job = make_test_job(job_id, Some(parent_id)); + windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true); + + let spans = state.span_exporter.get_finished_spans().unwrap(); + let found = spans.iter().any(|s| s.name == "full_job"); + assert!(!found, "child job should not produce a span"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_root_job_span_attributes_values() { + let state = ensure_setup().await; + state.span_exporter.reset(); + + let job_id = uuid::Uuid::new_v4(); + let job = make_test_job(job_id, None); + windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true); + + let spans = state.span_exporter.get_finished_spans().unwrap(); + let span = spans + .iter() + .find(|s| s.name == "full_job") + .expect("full_job span not found"); + + let get_attr = |key: &str| -> String { + span.attributes + .iter() + .find(|kv| kv.key.as_str() == key) + .map(|kv| kv.value.as_str().to_string()) + .unwrap_or_default() + }; + + assert_eq!(get_attr("job_id"), job_id.to_string()); + assert_eq!(get_attr("workspace_id"), "test-workspace"); + assert_eq!(get_attr("script_path"), "f/test/script"); +} diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 254ab33153..61f7289c0e 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -1081,6 +1081,115 @@ echo "$result" Ok(()) } +#[cfg(feature = "rlang")] +#[sqlx::test(fixtures("base"))] +async fn test_r_job(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +main <- function(msg) { + return(paste("hello", msg)) +} +"# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::Rlang, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + })) + .arg("msg", json!("world")) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, json!("hello world")); + Ok(()) +} + +#[cfg(feature = "rlang")] +#[sqlx::test(fixtures("base", "wmill_cli_test"))] +async fn test_r_get_variable(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +main <- function() { + return(get_variable("u/test-user/test_var")) +} +"# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::Rlang, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + })) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, json!("hello from variable")); + Ok(()) +} + +#[cfg(feature = "rlang")] +#[sqlx::test(fixtures("base", "wmill_cli_test"))] +async fn test_r_get_resource(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +main <- function() { + return(get_resource("u/test-user/test_res")) +} +"# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::Rlang, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + })) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, json!({"host": "localhost", "port": 5432})); + Ok(()) +} + #[cfg(feature = "nu")] #[sqlx::test(fixtures("base"))] async fn test_nu_job(db: Pool) -> anyhow::Result<()> { diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs index 8e3a4c7822..b5b04cb3e8 100644 --- a/backend/windmill-api-auth/src/auth.rs +++ b/backend/windmill-api-auth/src/auth.rs @@ -1,7 +1,6 @@ #[cfg(feature = "enterprise")] use crate::ee_oss::ExternalJwks; use axum::{ - async_trait, extract::{FromRequestParts, OriginalUri, Query}, Extension, Json, }; @@ -226,7 +225,15 @@ impl AuthCache { t_hash, w_id.as_ref(), ) - .map(|x| (x.owner, x.email, x.super_admin, x.scopes, x.label)) + .map(|x| { + ( + x.owner, + x.email, + x.super_admin, + x.scopes, + x.label, + ) + }) .fetch_optional(&self.db) .await .ok() @@ -235,7 +242,13 @@ impl AuthCache { if let Some(user) = user_o { let authed_o = { match user { - (Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => { + ( + Some(owner), + Some(email), + super_admin, + _, + label, + ) if w_id.is_some() => { let username_override = username_override_from_label(label); if let Some((prefix, name)) = owner.split_once('/') { if prefix == "u" { @@ -451,7 +464,11 @@ pub(crate) async fn extract_token(parts: &mut Parts, state: &S) None => Extension::::from_request_parts(parts, state) .await .ok() - .and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())), + .and_then(|cookies| { + cookies + .get(COOKIE_NAME) + .map(|c| c.value_trimmed().to_owned()) + }), }; #[derive(Deserialize)] @@ -504,7 +521,6 @@ impl BruteForceCounter { } } -#[async_trait] impl FromRequestParts for Tokened where S: Send + Sync, @@ -535,7 +551,6 @@ where } } -#[async_trait] impl FromRequestParts for OptTokened where S: Send + Sync, diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index aceef77e01..4e230c019b 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -12,8 +12,7 @@ pub mod ee; pub mod ee_oss; pub mod scopes; -use axum::async_trait; -use axum::extract::FromRequestParts; +use axum::extract::{FromRequestParts, OptionalFromRequestParts}; use http::request::Parts; use windmill_audit::audit_oss::AuditAuthorable; @@ -345,7 +344,6 @@ pub async fn maybe_refresh_folders( // ------------ FromRequestParts impls (direct call to auth module) ------------ -#[async_trait] impl FromRequestParts for ApiAuthed where S: Send + Sync, @@ -361,7 +359,24 @@ where } } -#[async_trait] +impl OptionalFromRequestParts for ApiAuthed +where + S: Send + Sync, +{ + type Rejection = std::convert::Infallible; + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result, Self::Rejection> { + Ok( + >::from_request_parts(parts, state) + .await + .ok(), + ) + } +} + impl FromRequestParts for OptJobAuthed where S: Send + Sync, @@ -397,7 +412,6 @@ fn empty_parts() -> Parts { #[derive(Clone, Debug)] pub struct OptAuthed(pub Option); -#[async_trait] impl FromRequestParts for OptAuthed where S: Send + Sync, @@ -408,7 +422,7 @@ where parts: &mut Parts, state: &S, ) -> std::result::Result { - ApiAuthed::from_request_parts(parts, state) + >::from_request_parts(parts, state) .await .map(|authed| Self(Some(authed))) .or_else(|_| Ok(Self(None))) diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index 26a2b3c74c..06ddbafcca 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -442,9 +442,22 @@ pub fn check_route_access( // Find the domain and kind for this route let (required_domain, required_kind, route_suffix) = extract_domain_from_route(route_path)?; - // Backward compatibility: MCP handlers expect unusual scope actions: all, favorites, hub. + // MCP scopes (mcp:all, mcp:favorites, mcp:hub:*, etc.) use a custom format + // that doesn't fit the standard domain:action model. Verify the token has at + // least one mcp: scope; MCP handlers do their own fine-grained checking. if required_domain == ScopeDomain::Mcp { - return Ok(()); + let is_scoped_token = token_scopes + .iter() + .any(|s| !s.starts_with("if_jobs:filter_tags:")); + if !is_scoped_token { + return Ok(()); + } + if token_scopes.iter().any(|s| s.starts_with("mcp:")) { + return Ok(()); + } + return Err(Error::NotAuthorized( + "Access denied. Required scope: mcp:*".to_string(), + )); } // tracing::error!("Checking route access {:?} {:?} {:?} {:?}", required_action, required_domain, required_kind, route_suffix); @@ -931,4 +944,50 @@ mod tests { ScopeDefinition::new("scripts", "read", None, Some(vec!["u/*".to_string()])); assert!(scope_specific_path.includes(&required_broad)); } + + #[test] + fn test_mcp_scope_bypass_blocked_without_mcp_scope() { + // A token with only jobs:read should NOT be able to access MCP endpoints + let scopes = vec!["jobs:read".to_string()]; + assert!(check_route_access(&scopes, "/api/w/test_workspace/mcp/something", "GET").is_err()); + assert!( + check_route_access(&scopes, "/api/w/test_workspace/mcp/something", "POST").is_err() + ); + } + + #[test] + fn test_mcp_scope_allowed_with_mcp_scope() { + // A token with mcp:all should access MCP endpoints + let scopes = vec!["mcp:all".to_string()]; + assert!(check_route_access(&scopes, "/api/w/test_workspace/mcp/something", "GET").is_ok()); + + // mcp:favorites should also work + let scopes = vec!["mcp:favorites".to_string()]; + assert!(check_route_access(&scopes, "/api/w/test_workspace/mcp/something", "POST").is_ok()); + + // mcp:scripts:path should also work + let scopes = vec!["mcp:scripts:u/admin/script1".to_string()]; + assert!(check_route_access(&scopes, "/api/w/test_workspace/mcp/something", "GET").is_ok()); + } + + #[test] + fn test_mcp_scope_filter_tags_only_treated_as_unrestricted() { + // Token with only filter_tags is not considered scoped — should be allowed + let scopes = vec!["if_jobs:filter_tags:tag1".to_string()]; + assert!(check_route_access(&scopes, "/api/w/test_workspace/mcp/something", "GET").is_ok()); + } + + #[test] + fn test_mcp_scope_mixed_scopes_without_mcp() { + // Token with multiple non-MCP scopes should be denied + let scopes = vec!["jobs:read".to_string(), "scripts:write".to_string()]; + assert!(check_route_access(&scopes, "/api/w/test_workspace/mcp/something", "GET").is_err()); + } + + #[test] + fn test_mcp_scope_mixed_scopes_with_mcp() { + // Token with MCP scope + other scopes should be allowed for MCP + let scopes = vec!["jobs:read".to_string(), "mcp:all".to_string()]; + assert!(check_route_access(&scopes, "/api/w/test_workspace/mcp/something", "GET").is_ok()); + } } diff --git a/backend/windmill-api-configs/src/lib.rs b/backend/windmill-api-configs/src/lib.rs index a05f2dbaa8..8485de5e95 100644 --- a/backend/windmill-api-configs/src/lib.rs +++ b/backend/windmill-api-configs/src/lib.rs @@ -28,11 +28,11 @@ use windmill_api_auth::{require_devops_role, ApiAuthed}; pub fn global_service() -> Router { Router::new() .route("/list_worker_groups", get(list_worker_groups)) - .route("/update/:name", post(update_config).delete(delete_config)) - .route("/get/:name", get(get_config)) + .route("/update/{name}", post(update_config).delete(delete_config)) + .route("/get/{name}", get(get_config)) .route("/list", get(list_configs)) .route( - "/list_autoscaling_events/:worker_group", + "/list_autoscaling_events/{worker_group}", get(list_autoscaling_events), ) .route( diff --git a/backend/windmill-api-flow-conversations/src/lib.rs b/backend/windmill-api-flow-conversations/src/lib.rs index bc37c9863d..70c96d1405 100644 --- a/backend/windmill-api-flow-conversations/src/lib.rs +++ b/backend/windmill-api-flow-conversations/src/lib.rs @@ -21,8 +21,8 @@ use windmill_common::{ pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_conversations)) - .route("/delete/:conversation_id", delete(delete_conversation)) - .route("/:conversation_id/messages", get(list_messages)) + .route("/delete/{conversation_id}", delete(delete_conversation)) + .route("/{conversation_id}/messages", get(list_messages)) } #[derive(Serialize, FromRow, Debug)] diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index e5a78da5b3..89fd9629b7 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -61,26 +61,26 @@ pub fn workspaced_service() -> Router { .route("/list", get(list_flows)) .route("/list_search", get(list_search_flows)) .route("/create", post(create_flow)) - .route("/update/*path", post(update_flow)) - .route("/archive/*path", post(archive_flow_by_path)) - .route("/delete/*path", delete(delete_flow_by_path)) - .route("/list_tokens/*path", get(list_tokens)) - .route("/get/*path", get(get_flow_by_path)) - .route("/deployment_status/p/*path", get(get_deployment_status)) - .route("/get/draft/*path", get(get_flow_by_path_w_draft)) - .route("/exists/*path", get(exists_flow_by_path)) + .route("/update/{*path}", post(update_flow)) + .route("/archive/{*path}", post(archive_flow_by_path)) + .route("/delete/{*path}", delete(delete_flow_by_path)) + .route("/list_tokens/{*path}", get(list_tokens)) + .route("/get/{*path}", get(get_flow_by_path)) + .route("/deployment_status/p/{*path}", get(get_deployment_status)) + .route("/get/draft/{*path}", get(get_flow_by_path_w_draft)) + .route("/exists/{*path}", get(exists_flow_by_path)) .route("/list_paths", get(list_paths)) - .route("/history/p/*path", get(get_flow_history)) - .route("/get_latest_version/*path", get(get_latest_version)) + .route("/history/p/{*path}", get(get_flow_history)) + .route("/get_latest_version/{*path}", get(get_latest_version)) .route( - "/list_paths_from_workspace_runnable/:runnable_kind/*path", + "/list_paths_from_workspace_runnable/{runnable_kind}/{*path}", get(list_paths_from_workspace_runnable), ) - .route("/history_update/v/:version", post(update_flow_history)) - .route("/get/v/:version", get(get_flow_version_by_id)) - .route("/get/v/:version/p/*path", get(get_flow_version)) + .route("/history_update/v/{version}", post(update_flow_history)) + .route("/get/v/{version}", get(get_flow_version_by_id)) + .route("/get/v/{version}/p/{*path}", get(get_flow_version)) .route( - "/toggle_workspace_error_handler/*path", + "/toggle_workspace_error_handler/{*path}", post(toggle_workspace_error_handler), ) } @@ -88,7 +88,7 @@ pub fn workspaced_service() -> Router { pub fn global_service() -> Router { Router::new() .route("/hub/list", get(list_hub_flows)) - .route("/hub/get/:id", get(get_hub_flow_by_id)) + .route("/hub/get/{id}", get(get_hub_flow_by_id)) } #[derive(Serialize, FromRow)] diff --git a/backend/windmill-api-groups/src/folder_history.rs b/backend/windmill-api-groups/src/folder_history.rs index b11f328e3d..8ce1086dbf 100644 --- a/backend/windmill-api-groups/src/folder_history.rs +++ b/backend/windmill-api-groups/src/folder_history.rs @@ -22,7 +22,7 @@ use serde::Serialize; use sqlx::FromRow; pub fn workspaced_service() -> Router { - Router::new().route("/get/:name", get(get_folder_permission_history)) + Router::new().route("/get/{name}", get(get_folder_permission_history)) } #[derive(Serialize, FromRow)] diff --git a/backend/windmill-api-groups/src/folders.rs b/backend/windmill-api-groups/src/folders.rs index 89a64621cc..ab1541adc8 100644 --- a/backend/windmill-api-groups/src/folders.rs +++ b/backend/windmill-api-groups/src/folders.rs @@ -40,14 +40,14 @@ pub fn workspaced_service() -> Router { .route("/list", get(list_folders)) .route("/listnames", get(list_foldernames)) .route("/create", post(create_folder)) - .route("/get/:name", get(get_folder)) - .route("/exists/:name", get(exists_folder)) - .route("/update/:name", post(update_folder)) - .route("/getusage/:name", get(get_folder_usage)) - .route("/delete/:name", delete(delete_folder)) - .route("/addowner/:name", post(add_owner)) - .route("/removeowner/:name", post(remove_owner)) - .route("/is_owner/*path", get(is_owner_api)) + .route("/get/{name}", get(get_folder)) + .route("/exists/{name}", get(exists_folder)) + .route("/update/{name}", post(update_folder)) + .route("/getusage/{name}", get(get_folder_usage)) + .route("/delete/{name}", delete(delete_folder)) + .route("/addowner/{name}", post(add_owner)) + .route("/removeowner/{name}", post(remove_owner)) + .route("/is_owner/{*path}", get(is_owner_api)) } #[derive(FromRow, Serialize, Deserialize, Clone)] diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index da9267419d..d7ea8418f9 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -48,9 +48,9 @@ const KINDS: [&str; 19] = [ pub fn workspaced_service() -> Router { Router::new() - .route("/get/*path", get(get_granular_acls)) - .route("/add/*path", post(add_granular_acl)) - .route("/remove/*path", post(remove_granular_acl)) + .route("/get/{*path}", get(get_granular_acls)) + .route("/add/{*path}", post(add_granular_acl)) + .route("/remove/{*path}", post(remove_granular_acl)) } #[derive(Serialize, Deserialize)] diff --git a/backend/windmill-api-groups/src/groups.rs b/backend/windmill-api-groups/src/groups.rs index ad0dde1781..af67f884c9 100644 --- a/backend/windmill-api-groups/src/groups.rs +++ b/backend/windmill-api-groups/src/groups.rs @@ -33,24 +33,24 @@ pub fn workspaced_service() -> Router { .route("/list", get(list_groups)) .route("/listnames", get(list_group_names)) .route("/create", post(create_group)) - .route("/get/:name", get(get_group)) - .route("/update/:name", post(update_group)) - .route("/delete/:name", delete(delete_group)) - .route("/adduser/:name", post(add_user)) - .route("/removeuser/:name", post(remove_user)) - .route("/is_owner/:name", get(is_owner)) + .route("/get/{name}", get(get_group)) + .route("/update/{name}", post(update_group)) + .route("/delete/{name}", delete(delete_group)) + .route("/adduser/{name}", post(add_user)) + .route("/removeuser/{name}", post(remove_user)) + .route("/is_owner/{name}", get(is_owner)) } pub fn global_service() -> Router { Router::new() .route("/list", get(list_igroups)) .route("/list_with_workspaces", get(list_igroups_with_workspaces)) - .route("/get/:name", get(get_igroup)) + .route("/get/{name}", get(get_igroup)) .route("/create", post(create_igroup)) - .route("/update/:name", post(update_igroup)) - .route("/delete/:name", delete(delete_igroup)) - .route("/adduser/:name", post(add_user_igroup)) - .route("/removeuser/:name", post(remove_user_igroup)) + .route("/update/{name}", post(update_igroup)) + .route("/delete/{name}", delete(delete_igroup)) + .route("/adduser/{name}", post(add_user_igroup)) + .route("/removeuser/{name}", post(remove_user_igroup)) .route("/export", get(export_igroups)) .route("/overwrite", post(overwrite_igroups)) } @@ -851,9 +851,21 @@ async fn add_user_igroup( #[cfg(all(feature = "private", feature = "enterprise"))] { use windmill_api_workspaces::workspaces_ee::auto_add_user; + use windmill_common::users::compute_highest_workspace_role; + + // Find all instance groups this user belongs to (includes the newly added group) + let user_igroups: Vec = sqlx::query_scalar!( + "SELECT igroup FROM email_to_igroup WHERE email = $1", + &email + ) + .fetch_all(&mut *tx) + .await?; + let workspaces = sqlx::query!( r#" - SELECT workspace_id, auto_invite->'instance_groups_roles' as instance_groups_roles + SELECT workspace_id, + auto_invite->'instance_groups_roles' as instance_groups_roles, + auto_invite->'instance_groups' as instance_groups_json FROM workspace_settings WHERE auto_invite->'instance_groups' ? $1 "#, @@ -861,34 +873,53 @@ async fn add_user_igroup( ) .fetch_all(&mut *tx) .await?; + for ws in workspaces { - let role = ws + let roles: std::collections::HashMap = ws .instance_groups_roles - .and_then(|r| r.get(&name).and_then(|v| v.as_str().map(String::from))) - .unwrap_or_else(|| "developer".to_string()); - let (is_admin, is_operator) = match role.as_str() { - "admin" => (true, false), - "operator" => (false, true), - _ => (false, false), - }; + .and_then(|r| serde_json::from_value(r).ok()) + .unwrap_or_default(); + + let ws_configured_groups: Vec = ws + .instance_groups_json + .and_then(|ig| serde_json::from_value(ig).ok()) + .unwrap_or_default(); + + let (best_group, is_admin, is_operator) = + compute_highest_workspace_role(&user_igroups, &ws_configured_groups, &roles); + + let instance_group_source = serde_json::json!({ + "source": "instance_group", + "group": &best_group + }); + + // auto_add_user creates the user if they don't exist (ON CONFLICT DO NOTHING). + // The operator flag here doesn't matter for the final state — the UPDATE below + // always sets the correct is_admin/operator based on the highest-precedence role. auto_add_user( &email, &ws.workspace_id, - &is_operator, + &false, &mut tx, &authed, - Some(serde_json::json!({"source": "instance_group", "group": &name})), + Some(instance_group_source.clone()), ) .await?; - if is_admin { - sqlx::query!( - "UPDATE usr SET is_admin = true WHERE workspace_id = $1 AND email = $2", - &ws.workspace_id, - &email - ) - .execute(&mut *tx) - .await?; - } + + // Set the correct role based on highest precedence across all groups. + // For new users, auto_add_user already stored added_via with source=instance_group, + // so this UPDATE will match. For existing instance_group users, it upgrades/corrects + // the role. Manually-added users (added_via is NULL or non-instance_group) are not affected. + sqlx::query!( + "UPDATE usr SET is_admin = $1, operator = $2, added_via = $3 WHERE workspace_id = $4 AND email = $5 AND added_via->>'source' = 'instance_group'", + is_admin, + is_operator, + &instance_group_source, + &ws.workspace_id, + &email + ) + .execute(&mut *tx) + .await?; } } diff --git a/backend/windmill-api-inputs/src/lib.rs b/backend/windmill-api-inputs/src/lib.rs index e253d02ea1..154e02e5a9 100644 --- a/backend/windmill-api-inputs/src/lib.rs +++ b/backend/windmill-api-inputs/src/lib.rs @@ -26,6 +26,7 @@ use windmill_common::{ jobs::JobKind, scripts::to_i64, utils::{not_found_if_none, paginate, Pagination}, + worker::CLOUD_HOSTED, }; pub fn workspaced_service() -> Router { Router::new() @@ -33,9 +34,9 @@ pub fn workspaced_service() -> Router { .route("/list", get(list_saved_inputs)) .route("/create", post(create_input)) .route("/update", post(update_input)) - .route("/delete/:id", post(delete_input)) + .route("/delete/{id}", post(delete_input)) .route( - "/:job_or_input_id/args", + "/{job_or_input_id}/args", get(get_args_from_history_or_saved_input), ) } @@ -134,11 +135,20 @@ async fn get_input_history( Query(g): Query, ) -> JsonResult> { let (per_page, offset) = paginate(pagination); + let per_page = if *CLOUD_HOSTED { + per_page.min(100) + } else { + per_page + }; let mut tx = user_db.begin(&authed).await?; let args_query = if let Some(args) = &g.args { - sql_builder::bind::Bind::bind(&"and v2_job.args @> ?", &args.replace("'", "''")) + if let Ok(v) = serde_json::from_str::(args) { + sql_builder::bind::Bind::bind(&"and v2_job.args @> ?", &v.to_string()) + } else { + "AND FALSE".to_string() + } } else { "".to_string() }; diff --git a/backend/windmill-api-integration-tests/Cargo.toml b/backend/windmill-api-integration-tests/Cargo.toml index cb4827e857..54f836657c 100644 --- a/backend/windmill-api-integration-tests/Cargo.toml +++ b/backend/windmill-api-integration-tests/Cargo.toml @@ -13,7 +13,7 @@ default = [] private = ["windmill-test-utils/private", "dep:aws-config", "dep:aws-credential-types", "dep:aws-sdk-sqs", "windmill-git-sync/private"] enterprise = ["windmill-test-utils/enterprise", "dep:base64", "windmill-git-sync/enterprise"] deno_core = ["windmill-test-utils/deno_core"] -mcp = [] +mcp = ["windmill-test-utils/mcp", "dep:rmcp"] run_inline = ["dep:windmill-worker", "windmill-test-utils/run_inline", "windmill-test-utils/duckdb"] [dependencies] @@ -40,3 +40,5 @@ aws-config = { workspace = true, optional = true } aws-credential-types = { workspace = true, optional = true } aws-sdk-sqs = { workspace = true, optional = true } base64 = { workspace = true, optional = true } +axum.workspace = true +rmcp = { workspace = true, optional = true } diff --git a/backend/windmill-api-integration-tests/tests/ai_routes.rs b/backend/windmill-api-integration-tests/tests/ai_routes.rs new file mode 100644 index 0000000000..543c3ed2a8 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/ai_routes.rs @@ -0,0 +1,106 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +/// Start a mock AI API that echoes back a valid chat completion response. +async fn start_mock_ai_api() -> u16 { + use axum::{routing::post, Json, Router}; + + let app = Router::new().fallback(post(|| async { + Json(json!({ + "id": "chatcmpl-test", + "object": "chat.completion", + "choices": [{"message": {"role": "assistant", "content": "hello"}}] + })) + })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + port +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_ai_proxy_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Start mock AI API + let mock_port = start_mock_ai_api().await; + let mock_url = format!("http://127.0.0.1:{mock_port}/v1"); + + // Create an openai resource pointing to the mock + let resp = authed( + client() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/resources/create" + )) + .json(&json!({ + "path": "f/ai/openai_config", + "resource_type": "openai", + "value": { + "api_key": "test-key", + "base_url": mock_url + } + })), + ) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "create openai resource", + ); + + // Set ai_config on workspace_settings directly via SQL + sqlx::query( + "UPDATE workspace_settings SET ai_config = $1::jsonb WHERE workspace_id = 'test-workspace'", + ) + .bind(json!({ + "providers": { + "openai": { + "resource_path": "f/ai/openai_config", + "models": ["gpt-4"] + } + } + })) + .execute(&db) + .await?; + + // POST /w/{ws}/ai/proxy/chat/completions + let resp = authed( + client() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/ai/proxy/chat/completions" + )) + .header("X-Provider", "openai") + .json(&json!({ + "model": "gpt-4", + "messages": [{"role": "user", "content": "hi"}] + })), + ) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "POST /ai/proxy/chat/completions", + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/audit.rs b/backend/windmill-api-integration-tests/tests/audit.rs new file mode 100644 index 0000000000..31994d8165 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/audit.rs @@ -0,0 +1,35 @@ +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_audit_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/audit"); + + // GET /list returns 200 (empty array) + let resp = authed(client().get(format!("{base}/list"))).send().await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /audit/list", + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/capture_unauthed.rs b/backend/windmill-api-integration-tests/tests/capture_unauthed.rs new file mode 100644 index 0000000000..f30a6e31b3 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/capture_unauthed.rs @@ -0,0 +1,83 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_capture_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // POST /capture/set_config → 200 (authed) + let resp = authed( + client() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/capture/set_config" + )) + .json(&json!({ + "trigger_kind": "webhook", + "path": "u/test-user/test_capture", + "is_flow": false + })), + ) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "POST /capture/set_config"); + + // GET /capture/list/{...} → 200 (authed) + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/capture/list/script/u/test-user/test_capture" + ))) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx( + status, + &body, + "GET /capture/list/script/u/test-user/test_capture", + ); + + // POST /capture/ping_config/{trigger_kind}/{runnable_kind}/{*path} → 200 + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/capture/ping_config/webhook/script/u/test-user/test_capture" + ))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "POST /capture/ping_config", + ); + + // GET /capture/get_configs/{runnable_kind}/{*path} → 200 + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/capture/get_configs/script/u/test-user/test_capture" + ))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /capture/get_configs", + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/concurrency_groups.rs b/backend/windmill-api-integration-tests/tests/concurrency_groups.rs new file mode 100644 index 0000000000..e4cf4ab8be --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/concurrency_groups.rs @@ -0,0 +1,48 @@ +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_concurrency_groups_2xx(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let resp = authed(client().get(format!( + "http://localhost:{port}/api/concurrency_groups/list" + ))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /api/concurrency_groups/list", + ); + + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/concurrency_groups/list_jobs" + ))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /api/w/test-workspace/concurrency_groups/list_jobs", + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/favorites.rs b/backend/windmill-api-integration-tests/tests/favorites.rs new file mode 100644 index 0000000000..888425eef7 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/favorites.rs @@ -0,0 +1,72 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_favorites_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + // Setup: create a script to favorite + let resp = authed(client().post(format!("{ws}/scripts/create"))) + .json(&json!({ + "path": "u/test-user/test_fav_script", + "summary": "test", + "description": "", + "content": "export function main() { return 1; }", + "language": "deno", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {}, + "required": [] + } + })) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "POST /scripts/create (setup)"); + + let fav_body = json!({ + "favorite_kind": "script", + "path": "u/test-user/test_fav_script" + }); + + // POST /favorites/star → 200 + let resp = authed(client().post(format!("{ws}/favorites/star"))) + .json(&fav_body) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "POST /favorites/star"); + + // POST /favorites/unstar → 200 + let resp = authed(client().post(format!("{ws}/favorites/unstar"))) + .json(&fav_body) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "POST /favorites/unstar"); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/folder_history.rs b/backend/windmill-api-integration-tests/tests/folder_history.rs new file mode 100644 index 0000000000..065977f386 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/folder_history.rs @@ -0,0 +1,51 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_folder_history_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Create a folder first + let resp = authed( + client() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/folders/create" + )) + .json(&json!({"name": "test_hist_folder", "owners": ["u/test-user"]})), + ) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "POST /folders/create"); + + // GET /folders_history/get/{folder} → 200 (empty array) + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/folders_history/get/test_hist_folder" + ))) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "GET /folders_history/get/test_hist_folder"); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/granular_acls.rs b/backend/windmill-api-integration-tests/tests/granular_acls.rs new file mode 100644 index 0000000000..c9cbbf0cf4 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/granular_acls.rs @@ -0,0 +1,54 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_granular_acls_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/acls"); + + // GET /acls/get/group_/all → 200 + let resp = authed(client().get(format!("{base}/get/group_/all"))) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "GET /acls/get/group_/all"); + + // POST /acls/add/group_/all → 200 + let resp = authed(client().post(format!("{base}/add/group_/all"))) + .json(&json!({"owner": "u/test-user-2", "write": true})) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "POST /acls/add/group_/all"); + + // POST /acls/remove/group_/all → 200 + let resp = authed(client().post(format!("{base}/remove/group_/all"))) + .json(&json!({"owner": "u/test-user-2"})) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "POST /acls/remove/group_/all"); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/group_history.rs b/backend/windmill-api-integration-tests/tests/group_history.rs new file mode 100644 index 0000000000..5a18901454 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/group_history.rs @@ -0,0 +1,32 @@ +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_group_history_2xx(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/groups_history"); + + let resp = authed(client().get(format!("{base}/get/all"))) + .send() + .await?; + assert_2xx(resp.status().as_u16(), &resp.text().await?, "GET /get/all"); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/health.rs b/backend/windmill-api-integration-tests/tests/health.rs new file mode 100644 index 0000000000..f624431510 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/health.rs @@ -0,0 +1,41 @@ +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_health_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/health"); + + // GET /health/status → 200 (no auth required) + let resp = client().get(format!("{base}/status")).send().await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "GET /health/status"); + + // GET /health/detailed → 200 (authed) + let resp = authed(client().get(format!("{base}/detailed"))) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "GET /health/detailed"); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/inputs.rs b/backend/windmill-api-integration-tests/tests/inputs.rs new file mode 100644 index 0000000000..6f806efdb5 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/inputs.rs @@ -0,0 +1,76 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_inputs_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/inputs"); + + // GET /history with fake runnable → 200 empty array + let resp = authed(client().get(format!( + "{base}/history?runnable_id=u/test-user/test&runnable_type=ScriptPath" + ))) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "GET /inputs/history"); + + // GET /list with fake runnable → 200 empty array + let resp = authed(client().get(format!( + "{base}/list?runnable_id=u/test-user/test&runnable_type=ScriptPath" + ))) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "GET /inputs/list"); + + // POST /create → 200, returns UUID + let resp = authed(client().post(format!( + "{base}/create?runnable_id=u/test-user/test&runnable_type=ScriptPath" + ))) + .json(&json!({"name": "test_input", "args": {}})) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "POST /inputs/create"); + let input_id: String = serde_json::from_str(&body)?; + + // GET /{id}/args → 200 + let resp = authed(client().get(format!("{base}/{input_id}/args"))) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "GET /inputs/{id}/args"); + + // POST /delete/{id} → 200 + let resp = authed(client().post(format!("{base}/delete/{input_id}"))) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "POST /inputs/delete/{id}"); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/job_metrics.rs b/backend/windmill-api-integration-tests/tests/job_metrics.rs new file mode 100644 index 0000000000..7408e4d3e3 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/job_metrics.rs @@ -0,0 +1,59 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +const FAKE_UUID: &str = "00000000-0000-0000-0000-000000000000"; + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_job_metrics_2xx(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/job_metrics"); + + let resp = authed(client().post(format!("{base}/get/{FAKE_UUID}"))) + .json(&json!({})) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "POST /get/{id}", + ); + + let resp = authed(client().post(format!("{base}/set_progress/{FAKE_UUID}"))) + .json(&json!({"percent": 50})) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "POST /set_progress/{id}", + ); + + let resp = authed(client().get(format!("{base}/get_progress/{FAKE_UUID}"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /get_progress/{id}", + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/jobs_authed.rs b/backend/windmill-api-integration-tests/tests/jobs_authed.rs new file mode 100644 index 0000000000..766c300d3d --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/jobs_authed.rs @@ -0,0 +1,309 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +fn assert_route_reachable(status: u16, body: &str, endpoint: &str) { + assert!( + status != 404 || !body.is_empty(), + "Router-level 404 for {endpoint}", + ); +} + +async fn insert_completed_job(db: &Pool) -> Uuid { + let id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, kind, tag, args) + VALUES ($1, 'test-workspace', 'test-user', 'u/test-user', 'script', 'deno', '{}'::jsonb)", + ) + .bind(id) + .execute(db) + .await + .unwrap(); + sqlx::query( + "INSERT INTO v2_job_completed (id, workspace_id, duration_ms, result, status) + VALUES ($1, 'test-workspace', 100, '42'::jsonb, 'success')", + ) + .bind(id) + .execute(db) + .await + .unwrap(); + id +} + +#[allow(dead_code)] +async fn create_script(port: u16) -> String { + let base = format!("http://localhost:{port}/api/w/test-workspace/scripts"); + let resp = authed(client().post(format!("{base}/create"))) + .json(&json!({ + "path": "u/test-user/test_job_script", + "summary": "test", + "description": "", + "content": "export function main() { return 42; }", + "language": "deno", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {}, + "required": [] + } + })) + .send() + .await + .unwrap(); + assert!( + resp.status().is_success(), + "create script: {}", + resp.status() + ); + "u/test-user/test_job_script".to_string() +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_jobs_authed_list_and_count(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/jobs"); + + // --- List/count endpoints (2xx with empty results) --- + + let resp = authed(client().get(format!("{base}/list"))).send().await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /jobs/list", + ); + + let resp = authed(client().get(format!("{base}/queue/list"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /jobs/queue/list", + ); + + let resp = authed(client().get(format!("{base}/queue/count"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /jobs/queue/count", + ); + + let resp = authed(client().get(format!("{base}/completed/list"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /jobs/completed/list", + ); + + let resp = authed(client().get(format!("{base}/completed/count"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /jobs/completed/count", + ); + + // --- Global endpoints --- + + let resp = client() + .get(format!("http://localhost:{port}/api/jobs/db_clock")) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /jobs/db_clock", + ); + + let resp = authed(client().get(format!( + "http://localhost:{port}/api/jobs/completed/count_by_tag" + ))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /jobs/completed/count_by_tag", + ); + + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_jobs_authed_completed_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/jobs"); + + let job_id = insert_completed_job(&db).await; + + let resp = authed(client().get(format!("{base}/completed/get/{job_id}"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /jobs/completed/get", + ); + + let resp = authed(client().get(format!("{base}/completed/get_result/{job_id}"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /jobs/completed/get_result", + ); + + let resp = authed(client().get(format!("{base}/completed/get_result_maybe/{job_id}"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /jobs/completed/get_result_maybe", + ); + + let resp = authed(client().get(format!("{base}/completed/get_timing/{job_id}"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /jobs/completed/get_timing", + ); + + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_jobs_authed_run_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/jobs"); + + // Run preview — no pre-existing script needed + let resp = authed(client().post(format!("{base}/run/preview"))) + .json(&json!({ + "content": "export function main() { return 1; }", + "language": "deno", + "args": {} + })) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "POST /jobs/run/preview", + ); + + // Run preview flow + let resp = authed(client().post(format!("{base}/run/preview_flow"))) + .json(&json!({ + "value": {"modules": []}, + "args": {} + })) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "POST /jobs/run/preview_flow", + ); + + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_jobs_authed_reachability(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/jobs"); + let fake = Uuid::nil(); + + // These need complex runtime but should hit the handler (not 404) + + let resp = authed(client().post(format!("{base}/flow/resume/{fake}"))) + .json(&json!({})) + .send() + .await?; + assert_route_reachable( + resp.status().as_u16(), + &resp.text().await?, + "POST /jobs/flow/resume", + ); + + let resp = authed(client().get(format!("{base}/job_signature/{fake}/1"))) + .send() + .await?; + assert_route_reachable( + resp.status().as_u16(), + &resp.text().await?, + "GET /jobs/job_signature", + ); + + let resp = authed(client().get(format!("{base}/resume_urls/{fake}/1"))) + .send() + .await?; + assert_route_reachable( + resp.status().as_u16(), + &resp.text().await?, + "GET /jobs/resume_urls", + ); + + let resp = authed(client().get(format!("{base}/result_by_id/{fake}/step1"))) + .send() + .await?; + assert_route_reachable( + resp.status().as_u16(), + &resp.text().await?, + "GET /jobs/result_by_id", + ); + + let resp = authed(client().post(format!("{base}/restart/f/{fake}"))) + .send() + .await?; + assert_route_reachable( + resp.status().as_u16(), + &resp.text().await?, + "POST /jobs/restart/f", + ); + + let resp = authed(client().post(format!("{base}/run/workflow_as_code/{fake}/main"))) + .json(&json!({})) + .send() + .await?; + assert_route_reachable( + resp.status().as_u16(), + &resp.text().await?, + "POST /jobs/run/workflow_as_code", + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/jobs_unauthed.rs b/backend/windmill-api-integration-tests/tests/jobs_unauthed.rs new file mode 100644 index 0000000000..fa8b6cb66f --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/jobs_unauthed.rs @@ -0,0 +1,250 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +/// Insert a minimal completed job directly into the database for testing. +async fn insert_completed_job(db: &Pool) -> Uuid { + let id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, kind, tag, args) + VALUES ($1, 'test-workspace', 'test-user', 'u/test-user', 'script', 'deno', '{}'::jsonb)", + ) + .bind(id) + .execute(db) + .await + .unwrap(); + + sqlx::query( + "INSERT INTO v2_job_completed (id, workspace_id, duration_ms, result, status) + VALUES ($1, 'test-workspace', 100, '42'::jsonb, 'success')", + ) + .bind(id) + .execute(db) + .await + .unwrap(); + + id +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_jobs_unauthed_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/jobs_u"); + + let job_id = insert_completed_job(&db).await; + + // --- No-data endpoints --- + + let resp = authed(client().post(format!("{base}/queue/get_started_at_by_ids"))) + .json(&json!([])) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "POST /queue/get_started_at_by_ids", + ); + + // --- Completed job endpoints (unauthed service, with auth header) --- + + let resp = authed(client().get(format!("{base}/get/{job_id}"))) + .send() + .await?; + assert_2xx(resp.status().as_u16(), &resp.text().await?, "GET /get"); + + let resp = authed(client().get(format!("{base}/get_logs/{job_id}"))) + .send() + .await?; + assert_2xx(resp.status().as_u16(), &resp.text().await?, "GET /get_logs"); + + let resp = authed(client().get(format!("{base}/get_completed_logs_tail/{job_id}"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /get_completed_logs_tail", + ); + + let resp = authed(client().get(format!("{base}/get_args/{job_id}"))) + .send() + .await?; + assert_2xx(resp.status().as_u16(), &resp.text().await?, "GET /get_args"); + + let resp = authed(client().get(format!("{base}/completed/get/{job_id}"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /completed/get", + ); + + let resp = authed(client().get(format!("{base}/completed/get_result/{job_id}"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /completed/get_result", + ); + + let resp = authed(client().get(format!("{base}/completed/get_result_maybe/{job_id}"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /completed/get_result_maybe", + ); + + let resp = authed(client().get(format!("{base}/completed/get_timing/{job_id}"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /completed/get_timing", + ); + + let resp = authed(client().get(format!("{base}/getupdate/{job_id}"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /getupdate", + ); + + Ok(()) +} + +const FAKE_UUID: &str = "00000000-0000-0000-0000-000000000000"; +const FAKE_SECRET: &str = "aabb"; + +/// Reachability tests for endpoints that need complex runtime. +/// These just verify the route matches (handler runs), not 2xx. +fn assert_route_reachable(status: u16, body: &str, endpoint: &str) { + assert!( + status != 404 || !body.is_empty(), + "Router-level 404 for {endpoint}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_jobs_unauthed_complex_reachability(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/jobs_u"); + + let resp = authed(client().get(format!("{base}/resume/{FAKE_UUID}/1/{FAKE_SECRET}"))) + .send() + .await?; + assert_route_reachable(resp.status().as_u16(), &resp.text().await?, "GET /resume"); + + let resp = authed(client().post(format!("{base}/cancel/{FAKE_UUID}/1/{FAKE_SECRET}"))) + .send() + .await?; + assert_route_reachable(resp.status().as_u16(), &resp.text().await?, "POST /cancel"); + + let resp = authed(client().get(format!("{base}/get_flow/{FAKE_UUID}/1/{FAKE_SECRET}"))) + .send() + .await?; + assert_route_reachable(resp.status().as_u16(), &resp.text().await?, "GET /get_flow"); + + let resp = authed(client().post(format!("{base}/queue/cancel/{FAKE_UUID}"))) + .json(&serde_json::json!({"reason": "test"})) + .send() + .await?; + assert_route_reachable( + resp.status().as_u16(), + &resp.text().await?, + "POST /queue/cancel", + ); + + let resp = authed(client().post(format!("{base}/queue/force_cancel/{FAKE_UUID}"))) + .json(&serde_json::json!({"reason": "test"})) + .send() + .await?; + assert_route_reachable( + resp.status().as_u16(), + &resp.text().await?, + "POST /queue/force_cancel", + ); + + let resp = authed(client().post(format!("{base}/flow/resume_suspended/{FAKE_UUID}"))) + .send() + .await?; + assert_route_reachable( + resp.status().as_u16(), + &resp.text().await?, + "POST /flow/resume_suspended", + ); + + let resp = authed(client().get(format!("{base}/flow/approval_info/{FAKE_UUID}"))) + .send() + .await?; + assert_route_reachable( + resp.status().as_u16(), + &resp.text().await?, + "GET /flow/approval_info", + ); + + let resp = authed(client().get(format!("{base}/get_root_job_id/{FAKE_UUID}"))) + .send() + .await?; + assert_route_reachable( + resp.status().as_u16(), + &resp.text().await?, + "GET /get_root_job_id", + ); + + let resp = authed(client().get(format!("{base}/get_flow_debug_info/{FAKE_UUID}"))) + .send() + .await?; + assert_route_reachable( + resp.status().as_u16(), + &resp.text().await?, + "GET /get_flow_debug_info", + ); + + let resp = authed(client().get(format!("{base}/get_log_file/{FAKE_UUID}/test.txt"))) + .send() + .await?; + assert_route_reachable( + resp.status().as_u16(), + &resp.text().await?, + "GET /get_log_file", + ); + + let resp = authed(client().post(format!("{base}/queue/cancel_persistent/u/test-user/fake"))) + .json(&serde_json::json!({"reason": "test"})) + .send() + .await?; + assert_route_reachable( + resp.status().as_u16(), + &resp.text().await?, + "POST /queue/cancel_persistent", + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/npm_proxy.rs b/backend/windmill-api-integration-tests/tests/npm_proxy.rs new file mode 100644 index 0000000000..9f3e813807 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/npm_proxy.rs @@ -0,0 +1,85 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +/// Start a mock npm registry that returns valid JSON for any GET request. +async fn start_mock_registry() -> u16 { + use axum::{routing::get, Json, Router}; + + let app = Router::new().fallback(get(|| async { + Json(json!({ + "name": "test-package", + "versions": {"1.0.0": {"name": "test-package", "version": "1.0.0"}}, + "dist-tags": {"latest": "1.0.0"} + })) + })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + port +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_npm_proxy_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/npm_proxy"); + + // Start mock npm registry + let mock_port = start_mock_registry().await; + let mock_url = format!("http://127.0.0.1:{mock_port}"); + + // Configure the npm registry to point to our mock + let resp = authed( + client() + .post(format!( + "http://localhost:{port}/api/settings/global/npm_config_registry" + )) + .json(&json!({"value": mock_url})), + ) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "POST /settings/global/npm_config_registry", + ); + + // GET /metadata/{package} + let resp = authed(client().get(format!("{base}/metadata/lodash"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /npm_proxy/metadata/lodash", + ); + + // GET /resolve/{package} + let resp = authed(client().get(format!("{base}/resolve/lodash"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /npm_proxy/resolve/lodash", + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/raw_apps.rs b/backend/windmill-api-integration-tests/tests/raw_apps.rs new file mode 100644 index 0000000000..97f90709b1 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/raw_apps.rs @@ -0,0 +1,33 @@ +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_raw_apps_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/raw_apps"); + + // GET /raw_apps/list → 200 (empty array) + let resp = authed(client().get(format!("{base}/list"))).send().await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "GET /raw_apps/list"); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/resources.rs b/backend/windmill-api-integration-tests/tests/resources.rs index 22dab5d418..363217712f 100644 --- a/backend/windmill-api-integration-tests/tests/resources.rs +++ b/backend/windmill-api-integration-tests/tests/resources.rs @@ -1,5 +1,7 @@ use serde_json::json; use sqlx::{Pool, Postgres}; +#[cfg(feature = "mcp")] +use uuid::Uuid; use windmill_test_utils::*; @@ -518,3 +520,182 @@ async fn test_mcp_tools(db: Pool) -> anyhow::Result<()> { Ok(()) } + +#[cfg(feature = "mcp")] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_mcp_endpoint_tools_list(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let resp = authed(client().get(format!( + "http://localhost:{port}/api/mcp/w/test-workspace/list_tools" + ))) + .send() + .await?; + assert_eq!(resp.status(), 200); + + let tools: Vec = resp.json().await?; + + let tool_names: Vec<&str> = tools.iter().filter_map(|t| t["name"].as_str()).collect(); + + assert!( + tool_names.contains(&"getJob"), + "getJob not found in MCP endpoint tools: {tool_names:?}" + ); + assert!( + tool_names.contains(&"getJobLogs"), + "getJobLogs not found in MCP endpoint tools: {tool_names:?}" + ); + + // Verify getJob has the expected path and method + let get_job_tool = tools.iter().find(|t| t["name"] == "getJob").unwrap(); + assert_eq!(get_job_tool["path"], "/w/{workspace}/jobs_u/get/{id}"); + assert_eq!(get_job_tool["method"], "GET"); + + // Verify getJobLogs has the expected path and method + let get_job_logs_tool = tools.iter().find(|t| t["name"] == "getJobLogs").unwrap(); + assert_eq!( + get_job_logs_tool["path"], + "/w/{workspace}/jobs_u/get_logs/{id}" + ); + assert_eq!(get_job_logs_tool["method"], "GET"); + + Ok(()) +} + +#[cfg(feature = "mcp")] +async fn insert_completed_job_with_logs(db: &Pool) -> Uuid { + let id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, kind, tag, args) + VALUES ($1, 'test-workspace', 'test-user', 'u/test-user', 'script', 'deno', '{}'::jsonb)", + ) + .bind(id) + .execute(db) + .await + .unwrap(); + + sqlx::query( + "INSERT INTO v2_job_completed (id, workspace_id, duration_ms, result, status) + VALUES ($1, 'test-workspace', 100, '42'::jsonb, 'success')", + ) + .bind(id) + .execute(db) + .await + .unwrap(); + + sqlx::query( + "INSERT INTO job_logs (job_id, workspace_id, logs, log_offset) + VALUES ($1, 'test-workspace', 'hello world test log', 0)", + ) + .bind(id) + .execute(db) + .await + .unwrap(); + + id +} + +#[cfg(feature = "mcp")] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_mcp_client_get_job_and_logs(db: Pool) -> anyhow::Result<()> { + use rmcp::model::{ + CallToolRequestParams, ClientCapabilities, ClientInfo, Implementation, + InitializeRequestParams, + }; + use rmcp::service::{RoleClient, RunningService}; + use rmcp::transport::streamable_http_client::{ + StreamableHttpClientTransport, StreamableHttpClientTransportConfig, + }; + use rmcp::ServiceExt; + + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start_mcp(db.clone()).await?; + let port = server.addr.port(); + + let job_id = insert_completed_job_with_logs(&db).await; + + // Create a token with MCP scopes + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) + VALUES (encode(sha256('MCP_TOKEN'::bytea), 'hex'), 'MCP_TOK', 'MCP_TOKEN', 'test@windmill.dev', 'mcp token', true, ARRAY['mcp:all'])", + ) + .execute(&db) + .await?; + + // Connect as MCP client + let config = StreamableHttpClientTransportConfig::with_uri(format!( + "http://localhost:{port}/api/mcp/w/test-workspace/mcp" + )) + .auth_header("MCP_TOKEN"); + let transport = StreamableHttpClientTransport::from_config(config); + + let client_info = ClientInfo { + protocol_version: Default::default(), + capabilities: ClientCapabilities::default(), + client_info: Implementation { + name: "test-client".to_string(), + title: None, + version: "0.0.1".to_string(), + description: None, + website_url: None, + icons: None, + }, + meta: None, + }; + + let client: RunningService = + client_info.serve(transport).await?; + + // --- Test getJob --- + let result = client + .call_tool(CallToolRequestParams { + name: "getJob".into(), + arguments: Some(serde_json::from_value(json!({ "id": job_id.to_string() }))?), + task: None, + meta: None, + }) + .await?; + + let text = result + .content + .first() + .and_then(|c| c.raw.as_text()) + .expect("getJob should return text content"); + let job: serde_json::Value = serde_json::from_str(&text.text)?; + assert_eq!(job["id"], job_id.to_string()); + assert_eq!(job["workspace_id"], "test-workspace"); + assert_eq!(job["created_by"], "test-user"); + assert_eq!(job["job_kind"], "script"); + assert!( + job["success"].as_bool().unwrap_or(false), + "job should be successful: {job}" + ); + + // --- Test getJobLogs --- + let result = client + .call_tool(CallToolRequestParams { + name: "getJobLogs".into(), + arguments: Some(serde_json::from_value(json!({ "id": job_id.to_string() }))?), + task: None, + meta: None, + }) + .await?; + + let text = result + .content + .first() + .and_then(|c| c.raw.as_text()) + .expect("getJobLogs should return text content"); + // The logs endpoint returns text/plain, which gets wrapped as a JSON string by call_endpoint + let logs: String = serde_json::from_str(&text.text)?; + assert!( + logs.contains("hello world test log"), + "expected logs to contain test log, got: {logs}" + ); + + client.cancel().await?; + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/scripts.rs b/backend/windmill-api-integration-tests/tests/scripts.rs index 74fd9c8611..f5e78f880f 100644 --- a/backend/windmill-api-integration-tests/tests/scripts.rs +++ b/backend/windmill-api-integration-tests/tests/scripts.rs @@ -108,7 +108,10 @@ async fn test_script_endpoints(db: Pool) -> anyhow::Result<()> { let resp = authed_get(port, "raw/p", "u/test-user/test_script.ts").await; assert_eq!(resp.status(), 200); let body = resp.text().await?; - assert!(body.contains("return 42"), "expected script content, got: {body}"); + assert!( + body.contains("return 42"), + "expected script content, got: {body}" + ); // --- raw by hash (requires .ts suffix) --- let resp = authed_get(port, "raw/h", &format!("{hash}.ts")).await; @@ -131,12 +134,10 @@ async fn test_script_endpoints(db: Pool) -> anyhow::Result<()> { assert!(list.iter().any(|s| s["path"] == "u/test-user/test_script")); // list with path_start filter - let resp = authed(client().get(format!( - "{base}/list?path_start=u/test-user/another" - ))) - .send() - .await - .unwrap(); + let resp = authed(client().get(format!("{base}/list?path_start=u/test-user/another"))) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200); let list = resp.json::>().await?; assert_eq!(list.len(), 1); @@ -233,12 +234,7 @@ async fn test_script_endpoints(db: Pool) -> anyhow::Result<()> { .send() .await .unwrap(); - assert_eq!( - resp.status(), - 200, - "history_update: {}", - resp.text().await? - ); + assert_eq!(resp.status(), 200, "history_update: {}", resp.text().await?); // --- toggle_workspace_error_handler (EE-gated, expect 400 in OSS) --- let resp = authed(client().post(script_url( @@ -268,22 +264,13 @@ async fn test_script_endpoints(db: Pool) -> anyhow::Result<()> { .send() .await .unwrap(); - assert_eq!( - resp.status(), - 200, - "tokened_raw: {}", - resp.text().await? - ); + assert_eq!(resp.status(), 200, "tokened_raw: {}", resp.text().await?); // --- archive by path --- - let resp = authed(client().post(script_url( - port, - "archive/p", - "u/test-user/another_script", - ))) - .send() - .await - .unwrap(); + let resp = authed(client().post(script_url(port, "archive/p", "u/test-user/another_script"))) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200); // archived script should still be gettable @@ -333,12 +320,10 @@ async fn test_script_endpoints(db: Pool) -> anyhow::Result<()> { // ===== Hub endpoints (require external network, expect 500 or 200) ===== // --- hub/top --- - let resp = authed(client().get(format!( - "http://localhost:{port}/api/scripts/hub/top" - ))) - .send() - .await - .unwrap(); + let resp = authed(client().get(format!("http://localhost:{port}/api/scripts/hub/top"))) + .send() + .await + .unwrap(); assert!( resp.status() == 200 || resp.status() == 500, "hub/top: unexpected status {}", @@ -372,12 +357,10 @@ async fn test_script_endpoints(db: Pool) -> anyhow::Result<()> { ); // --- integrations hub/list --- - let resp = authed(client().get(format!( - "http://localhost:{port}/api/integrations/hub/list" - ))) - .send() - .await - .unwrap(); + let resp = authed(client().get(format!("http://localhost:{port}/api/integrations/hub/list"))) + .send() + .await + .unwrap(); assert!( resp.status() == 200 || resp.status() == 500, "integrations hub/list: unexpected status {}", @@ -386,3 +369,97 @@ async fn test_script_endpoints(db: Pool) -> anyhow::Result<()> { Ok(()) } + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_auto_parent_resolves_parent_hash(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/scripts"); + + // Create v1 + let resp = authed(client().post(format!("{base}/create"))) + .json(&new_script( + "u/test-user/auto_parent_test", + "v1", + "export async function main() { return 1; }", + )) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "create v1: {}", resp.text().await?); + + // Get the hash of v1 + let resp = authed_get(port, "get/p", "u/test-user/auto_parent_test").await; + let body = resp.json::().await?; + let v1_hash = body["hash"].as_str().unwrap().to_string(); + + // Create v2 using auto_parent (no parent_hash provided) + let mut v2 = new_script( + "u/test-user/auto_parent_test", + "v2", + "export async function main() { return 2; }", + ); + v2["auto_parent"] = json!(true); + let resp = authed(client().post(format!("{base}/create"))) + .json(&v2) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 201, + "create v2 with auto_parent: {}", + resp.text().await? + ); + + // Get v2 and verify its parent_hash points to v1 + let resp = authed_get(port, "get/p", "u/test-user/auto_parent_test").await; + let body = resp.json::().await?; + assert_eq!(body["summary"], "v2"); + let v2_hash = body["hash"].as_str().unwrap().to_string(); + assert_ne!(v2_hash, v1_hash); + + // v2's parent_hashes should contain v1 + let parent_hashes = body["parent_hashes"].as_array().unwrap(); + assert!( + parent_hashes + .iter() + .any(|h| h.as_str() == Some(v1_hash.as_str())), + "v2 parent_hashes should contain v1 hash {v1_hash}, got: {parent_hashes:?}" + ); + + // Create v3 with auto_parent to confirm it chains correctly + let mut v3 = new_script( + "u/test-user/auto_parent_test", + "v3", + "export async function main() { return 3; }", + ); + v3["auto_parent"] = json!(true); + let resp = authed(client().post(format!("{base}/create"))) + .json(&v3) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 201, + "create v3 with auto_parent: {}", + resp.text().await? + ); + + let resp = authed_get(port, "get/p", "u/test-user/auto_parent_test").await; + let body = resp.json::().await?; + assert_eq!(body["summary"], "v3"); + + // v3's parent_hashes should contain v2 (and transitively v1) + let parent_hashes = body["parent_hashes"].as_array().unwrap(); + assert!( + parent_hashes + .iter() + .any(|h| h.as_str() == Some(v2_hash.as_str())), + "v3 parent_hashes should contain v2 hash {v2_hash}, got: {parent_hashes:?}" + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/sensitive_log_masking.rs b/backend/windmill-api-integration-tests/tests/sensitive_log_masking.rs index fb759ee4b5..d174f3759b 100644 --- a/backend/windmill-api-integration-tests/tests/sensitive_log_masking.rs +++ b/backend/windmill-api-integration-tests/tests/sensitive_log_masking.rs @@ -215,8 +215,8 @@ export async function main() { "scenario 1: secret value leaked in logs\nLogs:\n{logs1}" ); assert!( - logs1.contains("The secret value is: alp*****"), - "scenario 1: expected masked output with first 3 chars\nLogs:\n{logs1}" + logs1.contains("The secret value is: alp*****k2m"), + "scenario 1: expected masked output with first 3 + last 3 chars\nLogs:\n{logs1}" ); assert!( logs1.contains("[windmill] secret value was masked for security reasons, use string transformations to display full value"), @@ -277,11 +277,11 @@ export async function main() { "scenario 3: secret2 leaked\nLogs:\n{logs3}" ); assert!( - logs3.contains("secret1=alp*****"), + logs3.contains("secret1=alp*****k2m"), "scenario 3: secret1 not masked\nLogs:\n{logs3}" ); assert!( - logs3.contains("secret2=bet*****"), + logs3.contains("secret2=bet*****n3p"), "scenario 3: secret2 not masked\nLogs:\n{logs3}" ); @@ -309,7 +309,7 @@ export async function main() { "scenario 4: secret leaked mid-string\nLogs:\n{logs4}" ); assert!( - logs4.contains("token=alp*****&user=bob&format=json"), + logs4.contains("token=alp*****k2m&user=bob&format=json"), "scenario 4: mid-string masking failed\nLogs:\n{logs4}" ); @@ -338,7 +338,7 @@ export async function main() { !logs5.contains(secret2), "scenario 5: secret leaked\nLogs:\n{logs5}" ); - let mask_count = logs5.matches("bet*****").count(); + let mask_count = logs5.matches("bet*****n3p").count(); assert!( mask_count >= 3, "scenario 5: expected >= 3 masked occurrences, found {mask_count}\nLogs:\n{logs5}" @@ -374,7 +374,7 @@ export async function main() { "scenario 6: encrypted password leaked\nLogs:\n{logs6}" ); assert!( - logs6.contains("password is: enc*****"), + logs6.contains("password is: enc*****q5r"), "scenario 6: encrypted password not masked\nLogs:\n{logs6}" ); @@ -403,7 +403,7 @@ export async function main() { "scenario 7: resource secret leaked\nLogs:\n{logs7}" ); assert!( - logs7.contains("db password: res*****"), + logs7.contains("db password: res*****7t2"), "scenario 7: resource secret not masked\nLogs:\n{logs7}" ); // Non-secret field should remain visible diff --git a/backend/windmill-api-integration-tests/tests/service_logs.rs b/backend/windmill-api-integration-tests/tests/service_logs.rs new file mode 100644 index 0000000000..0c66916053 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/service_logs.rs @@ -0,0 +1,36 @@ +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_service_logs_2xx(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/service_logs"); + + let resp = authed(client().get(format!("{base}/list_files"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /list_files", + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/settings.rs b/backend/windmill-api-integration-tests/tests/settings.rs new file mode 100644 index 0000000000..8f21ffb483 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/settings.rs @@ -0,0 +1,116 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_settings_2xx(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/settings"); + + let resp = authed(client().get(format!("{base}/envs"))).send().await?; + assert_2xx(resp.status().as_u16(), &resp.text().await?, "GET /envs"); + + let resp = authed(client().get(format!("{base}/global/hub_base_url"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /global/hub_base_url", + ); + + let resp = authed(client().post(format!("{base}/global/test_key"))) + .json(&json!({"value": "test"})) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "POST /global/test_key", + ); + + let resp = authed(client().get(format!("{base}/instance_config"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /instance_config", + ); + + let resp = authed(client().get(format!("{base}/instance_config/yaml"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /instance_config/yaml", + ); + + let resp = authed(client().get(format!("{base}/latest_key_renewal_attempt"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /latest_key_renewal_attempt", + ); + + let resp = authed(client().post(format!("{base}/sync_cached_resource_types"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "POST /sync_cached_resource_types", + ); + + // --- Reachability only (need external services) --- + + let resp = authed( + client() + .post(format!("{base}/test_smtp")) + .json(&json!({"to": "test@test.com", "subject": "test", "content": "test"})), + ) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert!( + status != 404 || !body.is_empty(), + "Router-level 404 for POST /test_smtp" + ); + + let resp = authed( + client() + .post(format!("{base}/test_license_key")) + .json(&json!({"license_key": "fake"})), + ) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert!( + status != 404 || !body.is_empty(), + "Router-level 404 for POST /test_license_key" + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/trash.rs b/backend/windmill-api-integration-tests/tests/trash.rs new file mode 100644 index 0000000000..df32a5b3fc --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/trash.rs @@ -0,0 +1,41 @@ +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_trash_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/trash"); + + // GET /trash/list → 200 (admin, empty array) + let resp = authed(client().get(format!("{base}/list"))).send().await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "GET /trash/list"); + + // POST /trash/empty → 200 (admin) + let resp = authed(client().post(format!("{base}/empty"))) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "POST /trash/empty"); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs index 72a102dfd8..9226d3f74a 100644 --- a/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs +++ b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs @@ -14,6 +14,7 @@ use serde_json::json; use sqlx::{Pool, Postgres}; use std::time::Duration; +#[allow(unused_imports)] use windmill_test_utils::*; /// Row shape for querying deployment callback jobs from v2_job_queue @@ -27,6 +28,7 @@ struct DeploymentCallbackJob { } /// Poll for deployment callback jobs in the queue for a given script path +#[allow(dead_code)] async fn get_deployment_callback_jobs( db: &Pool, script_path: &str, @@ -63,6 +65,7 @@ async fn get_deployment_callback_jobs( } /// Configure git sync for the test workspace with workspace dependencies enabled +#[allow(dead_code)] async fn setup_git_sync_config(db: &Pool, sync_script_path: &str) -> anyhow::Result<()> { let git_sync_config = json!({ "include_type": ["workspacedependencies"], @@ -87,6 +90,7 @@ async fn setup_git_sync_config(db: &Pool, sync_script_path: &str) -> a } /// Create a git repository resource for testing +#[allow(dead_code)] async fn create_git_repo_resource(db: &Pool) -> anyhow::Result<()> { sqlx::query( r#" @@ -107,6 +111,7 @@ async fn create_git_repo_resource(db: &Pool) -> anyhow::Result<()> { } /// Create a dummy sync script for testing (with version >= 28103 for debouncing support) +#[allow(dead_code)] async fn create_sync_script(db: &Pool, path: &str) -> anyhow::Result { let hash: i64 = rand::random::().unsigned_abs() as i64; sqlx::query( @@ -126,6 +131,7 @@ async fn create_sync_script(db: &Pool, path: &str) -> anyhow::Result, name: &str) -> anyhow::Result<()> { sqlx::query( r#" diff --git a/backend/windmill-api-integration-tests/tests/workspace_deps.rs b/backend/windmill-api-integration-tests/tests/workspace_deps.rs new file mode 100644 index 0000000000..dcfe8877dd --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/workspace_deps.rs @@ -0,0 +1,39 @@ +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_workspace_deps_2xx(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspace_dependencies"); + + let resp = authed(client().get(format!("{base}/list"))).send().await?; + assert_2xx(resp.status().as_u16(), &resp.text().await?, "GET /list"); + + let resp = authed(client().get(format!("{base}/get_latest/python3"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "GET /get_latest/python3", + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/workspaces.rs b/backend/windmill-api-integration-tests/tests/workspaces.rs index 55e00eb05d..131cfbbae5 100644 --- a/backend/windmill-api-integration-tests/tests/workspaces.rs +++ b/backend/windmill-api-integration-tests/tests/workspaces.rs @@ -599,59 +599,60 @@ async fn test_workspace_endpoints(db: Pool) -> anyhow::Result<()> { .unwrap(); assert_eq!(resp.status(), 200, "tarball: {}", resp.status()); - // ===== Fork operations (on the newly created workspace) ===== + // ===== Fork operations (EE-only: CE limits workspace count to 2) ===== + #[cfg(feature = "enterprise")] + { + let new_ws_base = format!("http://localhost:{port}/api/w/new-test-ws/workspaces"); + let resp = authed(client().post(format!("{new_ws_base}/create_fork"))) + .json(&json!({ + "id": "wm-fork-test-ws", + "name": "Forked Test Workspace" + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "create_fork: {}", resp.text().await?); - // --- create_fork (workspace-scoped, from new-test-ws) --- - let new_ws_base = format!("http://localhost:{port}/api/w/new-test-ws/workspaces"); - let resp = authed(client().post(format!("{new_ws_base}/create_fork"))) - .json(&json!({ - "id": "wm-fork-test-ws", - "name": "Forked Test Workspace" - })) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), 200, "create_fork: {}", resp.text().await?); + // verify fork exists + let resp = authed(client().post(format!("{global_base}/exists"))) + .json(&json!({"id": "wm-fork-test-ws"})) + .send() + .await + .unwrap(); + assert_eq!(resp.json::().await?, true); - // verify fork exists - let resp = authed(client().post(format!("{global_base}/exists"))) - .json(&json!({"id": "wm-fork-test-ws"})) - .send() - .await - .unwrap(); - assert_eq!(resp.json::().await?, true); + // --- change_workspace_id --- + let fork_ws_base = format!("http://localhost:{port}/api/w/wm-fork-test-ws/workspaces"); + let resp = authed(client().post(format!("{fork_ws_base}/change_workspace_id"))) + .json(&json!({ + "new_id": "wm-fork-renamed", + "new_name": "Renamed Fork" + })) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "change_workspace_id: {}", + resp.text().await? + ); - // --- change_workspace_id --- - let fork_ws_base = format!("http://localhost:{port}/api/w/wm-fork-test-ws/workspaces"); - let resp = authed(client().post(format!("{fork_ws_base}/change_workspace_id"))) - .json(&json!({ - "new_id": "wm-fork-renamed", - "new_name": "Renamed Fork" - })) - .send() - .await - .unwrap(); - assert_eq!( - resp.status(), - 200, - "change_workspace_id: {}", - resp.text().await? - ); + // verify renamed workspace exists + let resp = authed(client().post(format!("{global_base}/exists"))) + .json(&json!({"id": "wm-fork-renamed"})) + .send() + .await + .unwrap(); + assert_eq!(resp.json::().await?, true); - // verify renamed workspace exists - let resp = authed(client().post(format!("{global_base}/exists"))) - .json(&json!({"id": "wm-fork-renamed"})) - .send() - .await - .unwrap(); - assert_eq!(resp.json::().await?, true); - - // clean up renamed fork - let resp = authed(client().delete(format!("{global_base}/delete/wm-fork-renamed"))) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), 200); + // clean up renamed fork + let resp = authed(client().delete(format!("{global_base}/delete/wm-fork-renamed"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + } // --- archive workspace (on the newly created one, not our main test workspace) --- let new_ws_base = format!("http://localhost:{port}/api/w/new-test-ws/workspaces"); @@ -803,3 +804,21 @@ async fn test_get_copilot_info_ignores_empty_instance_ai_row( Ok(()) } + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_get_imports(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + let resp = authed(client().get(format!("{base}/get_imports/u/test-user/nonexistent_script"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let imports = resp.json::>().await?; + assert!(imports.is_empty()); + + Ok(()) +} diff --git a/backend/windmill-api-jobs/src/concurrency_groups.rs b/backend/windmill-api-jobs/src/concurrency_groups.rs index 33f6045e49..f3ef3b14a8 100644 --- a/backend/windmill-api-jobs/src/concurrency_groups.rs +++ b/backend/windmill-api-jobs/src/concurrency_groups.rs @@ -25,8 +25,8 @@ use uuid::Uuid; pub fn global_service() -> Router { Router::new() .route("/list", get(list_concurrency_groups)) - .route("/prune/*concurrency_key", delete(prune_concurrency_group)) - .route("/:job_id/key", get(get_concurrency_key)) + .route("/prune/{*concurrency_key}", delete(prune_concurrency_group)) + .route("/{job_id}/key", get(get_concurrency_key)) } pub fn workspaced_service() -> Router { diff --git a/backend/windmill-api-jobs/src/job_metrics.rs b/backend/windmill-api-jobs/src/job_metrics.rs index ad316150c0..59cb23deff 100644 --- a/backend/windmill-api-jobs/src/job_metrics.rs +++ b/backend/windmill-api-jobs/src/job_metrics.rs @@ -22,13 +22,13 @@ pub fn workspaced_service() -> Router { .allow_origin(Any); Router::new() - .route("/get/:id", post(get_job_metrics).layer(cors.clone())) + .route("/get/{id}", post(get_job_metrics).layer(cors.clone())) .route( - "/set_progress/:id", + "/set_progress/{id}", post(set_job_progress).layer(cors.clone()), ) .route( - "/get_progress/:id", + "/get_progress/{id}", get(get_job_progress).layer(cors.clone()), ) } diff --git a/backend/windmill-api-jobs/src/query.rs b/backend/windmill-api-jobs/src/query.rs index b1b3fb4f0a..0d7503b5bc 100644 --- a/backend/windmill-api-jobs/src/query.rs +++ b/backend/windmill-api-jobs/src/query.rs @@ -8,6 +8,7 @@ //! Query builders for filtering job lists (queue and completed). +use serde_json; use sql_builder::prelude::*; use sql_builder::SqlBuilder; use windmill_common::utils::{escape_ilike_pattern, paginate_without_limits, Pagination}; @@ -200,7 +201,11 @@ pub fn filter_list_queue_query( } if let Some(args) = &lq.args { - sqlb.and_where("args @> ?".bind(&args.replace("'", "''"))); + if let Ok(v) = serde_json::from_str::(args) { + sqlb.and_where("args @> ?".bind(&v.to_string())); + } else { + sqlb.and_where("FALSE"); + } } if lq.scheduled_for_before_now.is_some_and(|x| x) { @@ -499,11 +504,19 @@ pub fn filter_list_completed_query( } if let Some(args) = &lq.args { - sqlb.and_where("args @> ?".bind(&args.replace("'", "''"))); + if let Ok(v) = serde_json::from_str::(args) { + sqlb.and_where("args @> ?".bind(&v.to_string())); + } else { + sqlb.and_where("FALSE"); + } } if let Some(result) = &lq.result { - sqlb.and_where("result @> ?".bind(&result.replace("'", "''"))); + if let Ok(v) = serde_json::from_str::(result) { + sqlb.and_where("result @> ?".bind(&v.to_string())); + } else { + sqlb.and_where("FALSE"); + } } if lq.is_not_schedule.unwrap_or(false) { diff --git a/backend/windmill-api-jobs/src/types.rs b/backend/windmill-api-jobs/src/types.rs index c59a54bd1c..f040e6eeb7 100644 --- a/backend/windmill-api-jobs/src/types.rs +++ b/backend/windmill-api-jobs/src/types.rs @@ -511,18 +511,14 @@ pub struct ResumeUrls { pub struct QueryOrBody(pub Option); -#[axum::async_trait] -impl FromRequest for QueryOrBody +impl FromRequest for QueryOrBody where D: DeserializeOwned, S: Send + Sync, { type Rejection = Response; - async fn from_request( - req: Request, - state: &S, - ) -> std::result::Result { + async fn from_request(req: Request, state: &S) -> std::result::Result { return if req.method() == axum::http::Method::GET { let Query(InPayload { payload }) = Query::from_request(req, state) .await diff --git a/backend/windmill-api-npm-proxy/src/lib.rs b/backend/windmill-api-npm-proxy/src/lib.rs index 903c2fb33c..25a3dd22d8 100644 --- a/backend/windmill-api-npm-proxy/src/lib.rs +++ b/backend/windmill-api-npm-proxy/src/lib.rs @@ -119,10 +119,10 @@ struct FileEntry { pub fn workspaced_service() -> Router { Router::new() // Use wildcards for package names to support scoped packages like @scope/package - .route("/metadata/*package", get(get_package_metadata)) - .route("/resolve/*package", get(resolve_package_version)) - .route("/filetree/*package_version", get(get_package_filetree)) - .route("/file/*package_version_filepath", get(get_package_file)) + .route("/metadata/{*package}", get(get_package_metadata)) + .route("/resolve/{*package}", get(resolve_package_version)) + .route("/filetree/{*package_version}", get(get_package_filetree)) + .route("/file/{*package_version_filepath}", get(get_package_file)) .layer( CorsLayer::new() .allow_origin(Any) diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index fc12ccc043..6e35b0f897 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -56,12 +56,12 @@ pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_schedule)) .route("/list_with_jobs", get(list_schedule_with_jobs)) - .route("/get/*path", get(get_schedule)) - .route("/exists/*path", get(exists_schedule)) + .route("/get/{*path}", get(get_schedule)) + .route("/exists/{*path}", get(exists_schedule)) .route("/create", post(create_schedule)) - .route("/update/*path", post(edit_schedule)) - .route("/delete/*path", delete(delete_schedule)) - .route("/setenabled/*path", post(set_enabled)) + .route("/update/{*path}", post(edit_schedule)) + .route("/delete/{*path}", delete(delete_schedule)) + .route("/setenabled/{*path}", post(set_enabled)) .route("/setdefaulthandler", post(set_default_error_handler)) // .route("/catchup/*path", post(do_catchup).get(list_catchup)) } @@ -658,7 +658,11 @@ async fn list_schedule( sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); } if let Some(args) = &lsq.args { - sqlb.and_where("args @> ?".bind(&args.replace("'", "''"))); + if let Ok(v) = serde_json::from_str::(args) { + sqlb.and_where("args @> ?".bind(&v.to_string())); + } else { + sqlb.and_where("FALSE"); + } } if let Some(path_start) = &lsq.path_start { sqlb.and_where_like_left("path", path_start); diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 2729357328..afa5ded9dc 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -190,18 +190,18 @@ impl ScriptWDraft { pub fn global_service() -> Router { Router::new() .route("/hub/top", get(get_top_hub_scripts)) - .route("/hub/get/*path", get(get_hub_script_by_path)) - .route("/hub/get_full/*path", get(get_full_hub_script_by_path)) - .route("/hub/pick/*path", get(pick_hub_script_by_path)) + .route("/hub/get/{*path}", get(get_hub_script_by_path)) + .route("/hub/get_full/{*path}", get(get_full_hub_script_by_path)) + .route("/hub/pick/{*path}", get(pick_hub_script_by_path)) } pub fn global_unauthed_service() -> Router { Router::new() .route( - "/tokened_raw/:workspace/:token/*path", + "/tokened_raw/{workspace}/{token}/{*path}", get(get_tokened_raw_script_by_path), ) - .route("/empty_ts/*path", get(get_empty_ts_script_by_path)) + .route("/empty_ts/{*path}", get(get_empty_ts_script_by_path)) } pub fn workspaced_service() -> Router { @@ -210,33 +210,33 @@ pub fn workspaced_service() -> Router { .route("/list_search", get(list_search_scripts)) .route("/create", post(create_script)) .route("/create_snapshot", post(create_snapshot_script)) - .route("/archive/p/*path", post(archive_script_by_path)) - .route("/get/draft/*path", get(get_script_by_path_w_draft)) - .route("/get/p/*path", get(get_script_by_path)) - .route("/list_tokens/*path", get(list_tokens)) - .route("/raw/p/*path", get(raw_script_by_path)) - .route("/raw_unpinned/p/*path", get(raw_script_by_path_unpinned)) - .route("/exists/p/*path", get(exists_script_by_path)) - .route("/archive/h/:hash", post(archive_script_by_hash)) - .route("/delete/h/:hash", post(delete_script_by_hash)) - .route("/delete/p/*path", post(delete_script_by_path)) + .route("/archive/p/{*path}", post(archive_script_by_path)) + .route("/get/draft/{*path}", get(get_script_by_path_w_draft)) + .route("/get/p/{*path}", get(get_script_by_path)) + .route("/list_tokens/{*path}", get(list_tokens)) + .route("/raw/p/{*path}", get(raw_script_by_path)) + .route("/raw_unpinned/p/{*path}", get(raw_script_by_path_unpinned)) + .route("/exists/p/{*path}", get(exists_script_by_path)) + .route("/archive/h/{hash}", post(archive_script_by_hash)) + .route("/delete/h/{hash}", post(delete_script_by_hash)) + .route("/delete/p/{*path}", post(delete_script_by_path)) .route("/delete_bulk", delete(delete_scripts_bulk)) - .route("/get/h/:hash", get(get_script_by_hash)) - .route("/raw/h/:hash", get(raw_script_by_hash)) - .route("/deployment_status/h/:hash", get(get_deployment_status)) + .route("/get/h/{hash}", get(get_script_by_hash)) + .route("/raw/h/{hash}", get(raw_script_by_hash)) + .route("/deployment_status/h/{hash}", get(get_deployment_status)) .route("/list_paths", get(list_paths)) .route( - "/toggle_workspace_error_handler/p/*path", + "/toggle_workspace_error_handler/p/{*path}", post(toggle_workspace_error_handler), ) - .route("/history/p/*path", get(get_script_history)) - .route("/get_latest_version/*path", get(get_latest_version)) + .route("/history/p/{*path}", get(get_script_history)) + .route("/get_latest_version/{*path}", get(get_latest_version)) .route( - "/list_paths_from_workspace_runnable/*path", + "/list_paths_from_workspace_runnable/{*path}", get(list_paths_from_workspace_runnable), ) .route( - "/history_update/h/:hash/p/*path", + "/history_update/h/{hash}/p/{*path}", post(update_script_history), ) .route("/list_dedicated_with_deps", get(list_dedicated_with_deps)) @@ -605,7 +605,7 @@ impl HandleDeploymentMetadata { } async fn create_script_internal<'c>( - ns: NewScript, + mut ns: NewScript, w_id: String, authed: ApiAuthed, db: sqlx::Pool, @@ -675,6 +675,17 @@ async fn create_script_internal<'c>( .to_owned(), )); }; + // When auto_parent is set, serialize concurrent creates for the same (workspace, path) + // so the clashing_script query always sees the latest committed head. + if ns.auto_parent.unwrap_or(false) { + sqlx::query_scalar!( + "SELECT pg_advisory_xact_lock(hashtext($1 || '/' || $2))", + &w_id, + &ns.path + ) + .fetch_one(&mut *tx) + .await?; + } let clashing_script = sqlx::query_as::<_, Script>( "SELECT * FROM script WHERE path = $1 AND archived = false AND workspace_id = $2", ) @@ -687,6 +698,15 @@ async fn create_script_internal<'c>( perms: serde_json::Value, p_path: String, } + // When auto_parent is set, resolve parent_hash to the current head for this path + // within the transaction. The advisory lock above ensures the second concurrent + // request waits until the first commits, so this query sees the updated head. + if ns.auto_parent.unwrap_or(false) { + if let Some(ref cs) = clashing_script { + ns.parent_hash = Some(cs.hash.clone()); + } + } + let parent_hashes_and_perms: Option = match (&ns.parent_hash, clashing_script) { (None, None) => Ok(None), (None, Some(s)) if !s.draft_only.unwrap_or(false) => Err(Error::BadRequest(format!( @@ -800,6 +820,7 @@ async fn create_script_internal<'c>( || ns.language == ScriptLang::Php || ns.language == ScriptLang::Java || ns.language == ScriptLang::Ruby + || ns.language == ScriptLang::Rlang // for related places search: ADD_NEW_LANG ) { Some(String::new()) @@ -1427,7 +1448,7 @@ async fn get_script_history( check_scopes(&authed, || format!("scripts:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let query_result = sqlx::query!( - "SELECT s.hash as hash, dm.deployment_msg as deployment_msg + "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash WHERE s.workspace_id = $1 AND s.path = $2 ORDER by s.created_at DESC", @@ -1443,6 +1464,7 @@ async fn get_script_history( .map(|row| ScriptHistory { script_hash: ScriptHash(row.hash), deployment_msg: row.deployment_msg, + created_at: Some(row.created_at), }) .collect(); return Ok(Json(result)); @@ -1457,7 +1479,7 @@ async fn get_latest_version( check_scopes(&authed, || format!("scripts:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let row_o = sqlx::query!( - "SELECT s.hash as hash, dm.deployment_msg as deployment_msg + "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash WHERE s.workspace_id = $1 AND s.path = $2 ORDER by s.created_at DESC LIMIT 1", @@ -1471,7 +1493,8 @@ async fn get_latest_version( if let Some(row) = row_o { let result = ScriptHistory { script_hash: ScriptHash(row.hash), - deployment_msg: row.deployment_msg, // + deployment_msg: row.deployment_msg, + created_at: Some(row.created_at), }; return Ok(Json(Some(result))); } else { diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index a1ea80f173..17954b1f11 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -58,7 +58,7 @@ pub fn global_service() -> Router { let r = Router::new() .route("/envs", get(get_local_settings)) .route( - "/global/:key", + "/global/{key}", post(set_global_setting).get(get_global_setting), ) .route("/list_global", get(list_global_settings)) @@ -80,7 +80,7 @@ pub fn global_service() -> Router { .route("/test_critical_channels", post(test_critical_channels)) .route("/critical_alerts", get(get_critical_alerts)) .route( - "/critical_alerts/:id/acknowledge", + "/critical_alerts/{id}/acknowledge", post(acknowledge_critical_alert), ) .route( @@ -92,7 +92,7 @@ pub fn global_service() -> Router { post(refresh_custom_instance_user_pwd), ) .route( - "/setup_custom_instance_pg_database/:name", + "/setup_custom_instance_pg_database/{name}", post(setup_custom_instance_pg_database), ) .route( diff --git a/backend/windmill-api-users/Cargo.toml b/backend/windmill-api-users/Cargo.toml index 13ab8143d8..c322d6cfda 100644 --- a/backend/windmill-api-users/Cargo.toml +++ b/backend/windmill-api-users/Cargo.toml @@ -21,6 +21,7 @@ windmill-api-auth.workspace = true windmill-audit.workspace = true windmill-git-sync.workspace = true +dashmap.workspace = true argon2.workspace = true axum.workspace = true chrono.workspace = true diff --git a/backend/windmill-api-users/src/lib.rs b/backend/windmill-api-users/src/lib.rs index 913bd46b82..ee5369e616 100644 --- a/backend/windmill-api-users/src/lib.rs +++ b/backend/windmill-api-users/src/lib.rs @@ -1 +1,4 @@ pub mod users; +#[cfg(feature = "private")] +pub mod users_ee; +mod users_oss; diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 8790a9c541..dcab4c62c9 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -12,6 +12,7 @@ use sqlx::{Postgres, Transaction}; use std::sync::atomic::AtomicBool; use std::sync::Arc; +use std::sync::LazyLock; use std::time::Duration; use windmill_api_auth::ApiAuthed; @@ -58,7 +59,44 @@ use windmill_common::{ use windmill_common::{BASE_URL, HUB_BASE_URL}; use windmill_git_sync::handle_deployment_metadata; -const COOKIE_PATH: &str = "/"; +pub const COOKIE_PATH: &str = "/"; + +const TOKEN_CREATE_LIMIT_PER_MINUTE: i32 = 10; + +struct TokenRateLimitEntry { + count: i32, + minute_bucket: i64, +} + +static TOKEN_CREATE_RATE_LIMIT: LazyLock> = + LazyLock::new(dashmap::DashMap::new); + +fn check_token_create_rate_limit(username: &str) -> Result<()> { + if !*CLOUD_HOSTED { + return Ok(()); + } + + let current_minute = chrono::Utc::now().timestamp() / 60; + + let mut entry = TOKEN_CREATE_RATE_LIMIT + .entry(username.to_string()) + .or_insert(TokenRateLimitEntry { count: 0, minute_bucket: current_minute }); + + if entry.minute_bucket != current_minute { + entry.count = 0; + entry.minute_bucket = current_minute; + } + + if entry.count >= TOKEN_CREATE_LIMIT_PER_MINUTE { + return Err(Error::Generic( + StatusCode::TOO_MANY_REQUESTS, + "Too many token creation requests. Please try again later.".to_string(), + )); + } + + entry.count += 1; + Ok(()) +} pub fn workspaced_service() -> Router { Router::new() @@ -66,32 +104,37 @@ pub fn workspaced_service() -> Router { .route("/list_usage", get(list_user_usage)) .route("/list_usernames", get(list_usernames)) .route("/exists", post(exists_username)) - .route("/get/:user", get(get_workspace_user)) - .route("/update/:user", post(update_workspace_user)) - .route("/delete/:user", delete(delete_workspace_user)) - .route("/convert_to_group/:user", post(convert_user_to_group)) - .route("/is_owner/*path", get(is_owner_of_path)) - .route("/whois/:username", get(whois)) + .route("/get/{user}", get(get_workspace_user)) + .route("/update/{user}", post(update_workspace_user)) + .route("/delete/{user}", delete(delete_workspace_user)) + .route("/convert_to_group/{user}", post(convert_user_to_group)) + .route("/is_owner/{*path}", get(is_owner_of_path)) + .route("/whois/{username}", get(whois)) .route("/whoami", get(whoami)) .route("/leave", post(leave_workspace)) - .route("/username_to_email/:username", get(username_to_email)) + .route("/username_to_email/{username}", get(username_to_email)) + .route( + "/impersonate_service_account", + post(impersonate_service_account), + ) + .route("/exit_impersonation", post(exit_impersonation)) } pub fn global_service() -> Router { Router::new() - .route("/exists/:email", get(exists_email)) + .route("/exists/{email}", get(exists_email)) .route("/email", get(get_email)) .route("/whoami", get(global_whoami)) .route("/list_invites", get(list_invites)) .route("/decline_invite", post(decline_invite)) .route("/accept_invite", post(accept_invite)) .route("/list_as_super_admin", get(list_users_as_super_admin)) - .route("/set_login_type/:user", post(set_login_type)) - .route("/update/:user", post(update_user)) - .route("/delete/:user", delete(delete_user)) - .route("/username_info/:user", get(get_instance_username_info)) + .route("/set_login_type/{user}", post(set_login_type)) + .route("/update/{user}", post(update_user)) + .route("/delete/{user}", delete(delete_user)) + .route("/username_info/{user}", get(get_instance_username_info)) .route("/tokens/create", post(create_token)) - .route("/tokens/delete/:token_prefix", delete(delete_token)) + .route("/tokens/delete/{token_prefix}", delete(delete_token)) .route("/tokens/list", get(list_tokens)) .route("/tokens/impersonate", post(impersonate)) .route("/usage", get(get_usage)) @@ -135,6 +178,7 @@ pub struct User { pub role: Option, #[serde(skip_serializing_if = "Option::is_none")] pub added_via: Option, + pub is_service_account: bool, } #[derive(Serialize)] @@ -176,6 +220,7 @@ pub struct UserInfo { pub folders: Vec, pub folders_owners: Vec, pub name: Option, + pub is_service_account: bool, } #[derive(FromRow, Serialize)] @@ -620,8 +665,9 @@ async fn is_valid_logout_redirect(rd: &str) -> bool { async fn whoami( Extension(db): Extension, Path(w_id): Path, - ApiAuthed { username, email, is_admin, groups, folders, .. }: ApiAuthed, + authed: ApiAuthed, ) -> JsonResult { + let ApiAuthed { username, email, is_admin, groups, folders, .. } = authed; let user = get_user(&w_id, &username, &db).await?; if let Some(user) = user { Ok(Json(user)) @@ -648,6 +694,7 @@ async fn whoami( .into_iter() .filter_map(|x| if x.2 { Some(x.0) } else { None }) .collect(), + is_service_account: false, })) } } @@ -663,11 +710,11 @@ async fn global_whoami( email = $1", email ) - .fetch_one(&db) + .fetch_optional(&db) .await - .map_err(|e| Error::internal_err(format!("fetching global identity: {e:#}"))); + .map_err(|e| Error::internal_err(format!("fetching global identity: {e:#}")))?; - if let Ok(user) = user { + if let Some(user) = user { Ok(Json(user)) } else if std::env::var("SUPERADMIN_SECRET").ok() == Some(token) { Ok(Json(GlobalUserInfo { @@ -685,7 +732,21 @@ async fn global_whoami( disabled: false, })) } else { - Err(user.unwrap_err()) + // Service accounts don't have a password row + Ok(Json(GlobalUserInfo { + email: email.clone(), + login_type: Some("service_account".to_string()), + super_admin: false, + devops: false, + verified: true, + name: None, + company: None, + username: None, + operator_only: Some(true), + first_time_user: false, + role_source: "service_account".to_string(), + disabled: false, + })) } } @@ -736,12 +797,13 @@ pub struct User2 { pub name: Option, #[serde(skip_serializing_if = "Option::is_none")] pub added_via: Option, + pub is_service_account: bool, } async fn get_user(w_id: &str, username: &str, db: &DB) -> Result> { let user = sqlx::query_as!( User2, - "SELECT usr.*, password.super_admin, password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2 + "SELECT usr.*, COALESCE(password.super_admin, false) as \"super_admin!\", password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2 ", username, w_id @@ -782,6 +844,7 @@ async fn get_user(w_id: &str, username: &str, db: &DB) -> Result, Extension(argon2): Extension>>, @@ -1731,8 +1795,10 @@ async fn login( return Ok("no_auth".to_string()); } - let mut tx = db.begin().await?; let email = email.to_lowercase(); + windmill_common::login_rate_limit::check_and_increment_login_attempt(&headers, &email)?; + + let mut tx = db.begin().await?; let audit_author = AuditAuthor { email: email.clone(), username: email.clone(), @@ -1764,6 +1830,7 @@ async fn login( None, ) .await?; + windmill_common::login_rate_limit::record_login_failure(&email); Err(Error::BadRequest("Invalid login".to_string())) } else { let token = create_session_token(&email, super_admin, &mut tx, cookies).await?; @@ -1800,6 +1867,7 @@ async fn login( None, ) .await?; + windmill_common::login_rate_limit::record_login_failure(&email); Err(Error::BadRequest("Invalid login".to_string())) } } @@ -1945,6 +2013,8 @@ async fn create_token( authed: ApiAuthed, Json(token_config): Json, ) -> Result<(StatusCode, String)> { + check_token_create_rate_limit(&authed.username)?; + let mut tx = db.begin().await?; let token = create_token_internal(&mut *tx, &db, &authed, token_config).await?; @@ -2025,6 +2095,44 @@ async fn impersonate( Ok((StatusCode::CREATED, token)) } +#[derive(Deserialize)] +pub struct ImpersonateServiceAccountRequest { + pub username: String, +} + +async fn impersonate_service_account( + Extension(db): Extension, + authed: ApiAuthed, + cookies: Cookies, + Tokened { token: current_token }: Tokened, + Path(w_id): Path, + Json(req): Json, +) -> Result<(StatusCode, String)> { + crate::users_oss::impersonate_service_account(db, authed, cookies, current_token, w_id, req) + .await +} + +#[derive(Deserialize)] +struct ExitImpersonationRequest { + token: String, +} + +async fn exit_impersonation( + cookies: Cookies, + Json(req): Json, +) -> Result { + let mut cookie = tower_cookies::Cookie::new(COOKIE_NAME, req.token); + cookie.set_secure(IS_SECURE.read().await.clone()); + cookie.set_same_site(Some(tower_cookies::cookie::SameSite::Lax)); + cookie.set_http_only(true); + cookie.set_path(COOKIE_PATH); + if COOKIE_DOMAIN.is_some() { + cookie.set_domain(COOKIE_DOMAIN.clone().unwrap()); + } + cookies.add(cookie); + Ok("exited impersonation".to_string()) +} + #[derive(Deserialize)] struct ListTokenQuery { exclude_ephemeral: Option, diff --git a/backend/windmill-api-users/src/users_oss.rs b/backend/windmill-api-users/src/users_oss.rs new file mode 100644 index 0000000000..a42cce8405 --- /dev/null +++ b/backend/windmill-api-users/src/users_oss.rs @@ -0,0 +1,28 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::users_ee::*; + +#[cfg(not(feature = "private"))] +use crate::users::ImpersonateServiceAccountRequest; +#[cfg(not(feature = "private"))] +use http::StatusCode; +#[cfg(not(feature = "private"))] +use tower_cookies::Cookies; +#[cfg(not(feature = "private"))] +use windmill_api_auth::ApiAuthed; +#[cfg(not(feature = "private"))] +use windmill_common::DB; + +#[cfg(not(feature = "private"))] +pub async fn impersonate_service_account( + _db: DB, + _authed: ApiAuthed, + _cookies: Cookies, + _current_token: String, + _w_id: String, + _req: ImpersonateServiceAccountRequest, +) -> windmill_common::error::Result<(StatusCode, String)> { + Err(windmill_common::error::Error::BadRequest( + "Service accounts require Windmill Enterprise Edition".to_string(), + )) +} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 4e832b3a93..cb5d9c2a03 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -75,10 +75,12 @@ pub fn workspaced_service() -> Router { .route("/archive", post(archive_workspace)) .route("/invite_user", post(invite_user)) .route("/add_user", post(add_user)) + .route("/create_service_account", post(create_service_account)) .route("/delete_invite", post(delete_invite)) .route("/rebuild_dependency_map", post(rebuild_dependency_map)) .route("/get_dependency_map", get(get_dependency_map)) - .route("/get_dependents/*imported_path", get(get_dependents)) + .route("/get_dependents/{*imported_path}", get(get_dependents)) + .route("/get_imports/{*importer_path}", get(get_imports)) .route("/get_dependents_amounts", post(get_dependents_amounts)) .route("/get_settings", get(get_settings)) .route( @@ -151,14 +153,14 @@ pub fn workspaced_service() -> Router { post(create_workspace_fork_branch), ) .route( - "/reset_diff_tally/:fork_workspace_id", + "/reset_diff_tally/{fork_workspace_id}", post(reset_workspace_diffs), ) - .route("/compare/:target_workspace_id", get(compare_workspaces)) + .route("/compare/{target_workspace_id}", get(compare_workspaces)) .route("/protection_rules", get(list_protection_rules)) .route("/protection_rules", post(create_protection_rule)) .route( - "/protection_rules/:rule_name", + "/protection_rules/{rule_name}", post(update_protection_rule).delete(delete_protection_rule), ) .route("/log_chat", post(log_ai_chat)) @@ -175,9 +177,9 @@ pub fn global_service() -> Router { .route("/exists", post(exists_workspace)) .route("/exists_username", post(exists_username)) .route("/allowed_domain_auto_invite", get(is_allowed_auto_domain)) - .route("/unarchive/:workspace", post(unarchive_workspace)) + .route("/unarchive/{workspace}", post(unarchive_workspace)) .route( - "/delete/:workspace", + "/delete/{workspace}", delete(crate::workspaces_extra::delete_workspace), ) .route( @@ -651,25 +653,23 @@ async fn get_settings( } async fn get_copilot_settings_state( - authed: ApiAuthed, + _authed: ApiAuthed, Path(w_id): Path, - Extension(user_db): Extension, + Extension(db): Extension, ) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; let workspace_ai_config = sqlx::query_scalar!( "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1", &w_id ) - .fetch_optional(&mut *tx) + .fetch_optional(&db) .await .map_err(|e| Error::internal_err(format!("getting workspace ai settings: {e:#}")))?; let workspace_ai_config = not_found_if_none(workspace_ai_config, "workspace settings", &w_id)?; let instance_ai_config: Option = sqlx::query_scalar("SELECT value FROM global_settings WHERE name = 'ai_config'") - .fetch_optional(&mut *tx) + .fetch_optional(&db) .await .map_err(|e| Error::internal_err(format!("getting instance ai settings: {e:#}")))?; - tx.commit().await?; Ok(Json(build_copilot_settings_state( has_ai_providers(workspace_ai_config.as_ref()), @@ -1134,6 +1134,12 @@ async fn edit_webhook( ) -> Result { require_admin(is_admin, &username)?; + if *CLOUD_HOSTED { + return Err(Error::BadRequest( + "Workspace webhooks are not available on cloud-hosted instances".to_string(), + )); + } + let mut tx = db.begin().await?; if let Some(webhook) = &ew.webhook { @@ -1690,6 +1696,18 @@ async fn check_git_sync_access(_db: &DB, _w_id: &str) -> Result<()> { Ok(()) } +// Anchor the CE-only query for `cargo sqlx prepare` (which runs with --features enterprise) +#[cfg(feature = "enterprise")] +#[allow(dead_code)] +async fn _sqlx_anchor_ce_user_count(db: &DB, w_id: &str) { + let _ = sqlx::query_scalar!( + "SELECT COUNT(*) FROM usr WHERE workspace_id = $1 AND disabled = false", + w_id + ) + .fetch_one(db) + .await; +} + #[cfg(not(feature = "enterprise"))] async fn check_git_sync_access(db: &DB, w_id: &str) -> Result<()> { let user_count: i64 = sqlx::query_scalar!( @@ -4221,6 +4239,20 @@ If you do not have an account on {}, login with SSO or ask an admin to create an )) } +#[derive(Deserialize)] +pub struct NewServiceAccount { + pub username: String, +} + +async fn create_service_account( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(nu): Json, +) -> Result<(StatusCode, String)> { + crate::workspaces_oss::create_service_account(authed, db, w_id, nu).await +} + async fn delete_invite( ApiAuthed { username, is_admin, .. }: ApiAuthed, Extension(db): Extension, @@ -4346,6 +4378,30 @@ async fn get_dependents( Ok(Json(dependents)) } +async fn get_imports( + Extension(db): Extension, + Path((w_id, importer_path)): Path<(String, String)>, + _authed: ApiAuthed, +) -> JsonResult> { + tracing::debug!( + workspace_id = %w_id, + importer_path = %importer_path, + "API: Getting imports for importer path" + ); + + let imports = ScopedDependencyMap::get_imports(&importer_path, &w_id, &db).await?; + + tracing::debug!( + workspace_id = %w_id, + importer_path = %importer_path, + imports_count = imports.len(), + "API: Found imports: {:?}", + imports + ); + + Ok(Json(imports)) +} + #[derive(Serialize, Debug)] struct DependentsAmount { imported_path: String, diff --git a/backend/windmill-api-workspaces/src/workspaces_oss.rs b/backend/windmill-api-workspaces/src/workspaces_oss.rs index da46622c26..872061e554 100644 --- a/backend/windmill-api-workspaces/src/workspaces_oss.rs +++ b/backend/windmill-api-workspaces/src/workspaces_oss.rs @@ -3,7 +3,9 @@ pub use crate::workspaces_ee::*; #[cfg(not(feature = "private"))] -use crate::workspaces::EditAutoInvite; +use crate::workspaces::{EditAutoInvite, NewServiceAccount}; +#[cfg(not(feature = "private"))] +use http::StatusCode; #[cfg(not(feature = "private"))] use windmill_api_auth::ApiAuthed; #[cfg(not(feature = "private"))] @@ -20,3 +22,15 @@ pub async fn edit_auto_invite( "Not implemented on OSS".to_string(), )) } + +#[cfg(not(feature = "private"))] +pub async fn create_service_account( + _authed: ApiAuthed, + _db: DB, + _w_id: String, + _nu: NewServiceAccount, +) -> windmill_common::error::Result<(StatusCode, String)> { + Err(windmill_common::error::Error::BadRequest( + "Service accounts require Windmill Enterprise Edition".to_string(), + )) +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 2b83550ab2..9e959e670f 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.665.0 + version: 1.672.0 title: Windmill API contact: @@ -2125,6 +2125,87 @@ paths: schema: type: string + /w/{workspace}/workspaces/create_service_account: + post: + summary: create a service account + operationId: createServiceAccount + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + username: + type: string + required: + - username + responses: + "201": + description: service account created + content: + text/plain: + schema: + type: string + + /w/{workspace}/users/impersonate_service_account: + post: + summary: impersonate a service account + operationId: impersonateServiceAccount + tags: + - user + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + username: + type: string + required: + - username + responses: + "201": + description: impersonation token + content: + text/plain: + schema: + type: string + + /w/{workspace}/users/exit_impersonation: + post: + summary: exit service account impersonation + operationId: exitImpersonation + tags: + - user + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + token: + type: string + required: + - token + responses: + "200": + description: exited impersonation + content: + text/plain: + schema: + type: string + /w/{workspace}/workspaces/delete_invite: post: summary: delete user invite @@ -2714,6 +2795,30 @@ paths: items: $ref: "#/components/schemas/DependencyDependent" + /w/{workspace}/workspaces/get_imports/{importer_path}: + get: + summary: get script imports for an importer path + operationId: getImports + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: importer_path + in: path + required: true + schema: + type: string + description: The script path to get imports for + responses: + "200": + description: list of imported script paths + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/workspaces/get_dependents_amounts: post: summary: get dependents amounts for multiple imported paths @@ -10493,6 +10598,7 @@ paths: get: summary: get job operationId: getJob + x-mcp-tool: true tags: - job parameters: @@ -10534,7 +10640,8 @@ paths: /w/{workspace}/jobs_u/get_logs/{id}: get: summary: get job logs - operationId: getJob logs + operationId: getJobLogs + x-mcp-tool: true tags: - job parameters: @@ -20099,6 +20206,8 @@ components: nullable: true allOf: - $ref: "#/components/schemas/UserSource" + is_service_account: + type: boolean required: - email - username @@ -20596,6 +20705,7 @@ components: nu, java, ruby, + rlang, duckdb, bunnative, # for related places search: ADD_NEW_LANG @@ -21707,6 +21817,13 @@ components: required: - key - value + filter_logic: + type: string + enum: + - and + - or + default: and + description: "Logic to apply when evaluating filters. 'and' requires all filters to match, 'or' requires any filter to match." initial_messages: type: array nullable: true @@ -21768,6 +21885,13 @@ components: required: - key - value + filter_logic: + type: string + enum: + - and + - or + default: and + description: "Logic to apply when evaluating filters. 'and' requires all filters to match, 'or' requires any filter to match." initial_messages: type: array nullable: true @@ -21836,6 +21960,13 @@ components: required: - key - value + filter_logic: + type: string + enum: + - and + - or + default: and + description: "Logic to apply when evaluating filters. 'and' requires all filters to match, 'or' requires any filter to match." initial_messages: type: array nullable: true @@ -22712,6 +22843,13 @@ components: required: - key - value + filter_logic: + type: string + enum: + - and + - or + default: and + description: "Logic to apply when evaluating filters. 'and' requires all filters to match, 'or' requires any filter to match." auto_offset_reset: type: string enum: @@ -22783,6 +22921,13 @@ components: required: - key - value + filter_logic: + type: string + enum: + - and + - or + default: and + description: "Logic to apply when evaluating filters. 'and' requires all filters to match, 'or' requires any filter to match." auto_offset_reset: type: string enum: @@ -22846,6 +22991,13 @@ components: required: - key - value + filter_logic: + type: string + enum: + - and + - or + default: and + description: "Logic to apply when evaluating filters. 'and' requires all filters to match, 'or' requires any filter to match." auto_offset_reset: type: string enum: diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index d77a0fa8cc..16cfb0166c 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -607,11 +607,11 @@ fn transform_fim_to_chat_completions(body: &Bytes) -> Result<(Bytes, String)> { } pub fn global_service() -> Router { - Router::new().route("/proxy/*ai", post(global_proxy).get(global_proxy)) + Router::new().route("/proxy/{*ai}", post(global_proxy).get(global_proxy)) } pub fn workspaced_service() -> Router { - let router = Router::new().route("/proxy/*ai", post(proxy).get(proxy)); + let router = Router::new().route("/proxy/{*ai}", post(proxy).get(proxy)); #[cfg(feature = "bedrock")] let router = router.route("/check_bedrock_credentials", get(check_bedrock_credentials)); diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 9ff74387d1..e9f262881a 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -79,52 +79,64 @@ use windmill_common::{jwt, oauth2::HmacSha256, variables::get_workspace_key}; #[cfg(feature = "parquet")] use windmill_types::s3::{S3Object, S3Permission}; -pub fn workspaced_service() -> Router { +pub fn workspaced_service(raw_app_body_limit: usize) -> Router { Router::new() .route("/list", get(list_apps)) .route("/list_search", get(list_search_apps)) - .route("/get/p/*path", get(get_app)) - .route("/get/lite/*path", get(get_app_lite)) - .route("/get/draft/*path", get(get_app_w_draft)) - .route("/secret_of/*path", get(get_secret_id)) + .route("/get/p/{*path}", get(get_app)) + .route("/get/lite/{*path}", get(get_app_lite)) + .route("/get/draft/{*path}", get(get_app_w_draft)) + .route("/secret_of/{*path}", get(get_secret_id)) .route( - "/secret_of_latest_version/*path", + "/secret_of_latest_version/{*path}", get(get_latest_version_secret_id), ) - .route("/get/v/*id", get(get_app_by_id)) - .route("/get_data/v/*id", get(get_raw_app_data)) - .route("/exists/*path", get(exists_app)) - .route("/update/*path", post(update_app)) - .route("/update_raw/*path", post(update_app_raw)) - .route("/delete/*path", delete(delete_app)) - .route("/create", post(create_app)) - .route("/create_raw", post(create_app_raw)) - .route("/history/p/*path", get(get_app_history)) - .route("/get_latest_version/*path", get(get_latest_version)) - .route("/history_update/a/:id/v/:version", post(update_app_history)) + .route("/get/v/{*id}", get(get_app_by_id)) + .route("/get_data/v/{*id}", get(get_raw_app_data)) + .route("/exists/{*path}", get(exists_app)) + .route("/update/{*path}", post(update_app)) .route( - "/list_paths_from_workspace_runnable/:runnable_kind/*path", + "/update_raw/{*path}", + post(update_app_raw).layer(axum::extract::DefaultBodyLimit::max(raw_app_body_limit)), + ) + .route("/delete/{*path}", delete(delete_app)) + .route("/create", post(create_app)) + .route( + "/create_raw", + post(create_app_raw).layer(axum::extract::DefaultBodyLimit::max(raw_app_body_limit)), + ) + .route("/history/p/{*path}", get(get_app_history)) + .route("/get_latest_version/{*path}", get(get_latest_version)) + .route( + "/history_update/a/{id}/v/{version}", + post(update_app_history), + ) + .route( + "/list_paths_from_workspace_runnable/{runnable_kind}/{*path}", get(list_paths_from_workspace_runnable), ) - .route("/custom_path_exists/*custom_path", get(custom_path_exists)) + .route( + "/custom_path_exists/{*custom_path}", + get(custom_path_exists), + ) .route("/sign_s3_objects", post(sign_s3_objects)) } pub fn unauthed_service() -> Router { Router::new() - .route("/execute_component/*path", post(execute_component)) - .route("/upload_s3_file/*path", post(upload_s3_file_from_app)) + .route("/execute_component/{*path}", post(execute_component)) + .route("/upload_s3_file/{*path}", post(upload_s3_file_from_app)) .route("/delete_s3_file", delete(delete_s3_file_from_app)) - .route("/download_s3_file/*path", get(download_s3_file_from_app)) - .route("/public_app/:secret", get(get_public_app_by_secret)) - .route("/public_resource/*path", get(get_public_resource)) - .route("/get_data/v/*id", get(get_raw_app_data)) + .route("/download_s3_file/{*path}", get(download_s3_file_from_app)) + .route("/public_app/{secret}", get(get_public_app_by_secret)) + .route("/public_resource/{*path}", get(get_public_resource)) + .route("/get_data/v/{*id}", get(get_raw_app_data)) } pub fn global_service() -> Router { Router::new() .route("/hub/list", get(list_hub_apps)) - .route("/hub/get/:id", get(get_hub_app_by_id)) - .route("/hub/get_raw/:id", get(get_hub_raw_app_by_id)) + .route("/hub/get/{id}", get(get_hub_app_by_id)) + .route("/hub/get_raw/{id}", get(get_hub_raw_app_by_id)) } #[derive(FromRow, Deserialize, Serialize)] @@ -1004,18 +1016,20 @@ macro_rules! process_app_multipart { let mut saved_app = None; let mut uploaded_js = false; + let request_size_limit_mb = *crate::REQUEST_SIZE_LIMIT.read().await / (1024 * 1024); + let raw_app_limit_mb = request_size_limit_mb * 5; let mut multipart = $multipart; while let Some(field) = multipart .next_field() .await - .map_err(|e| Error::BadRequest(format!("failed to read multipart field: {e}")))? + .map_err(|e| Error::BadRequest(format!("failed to read multipart field: {e}. Could be due to the request size limit for raw app bundles which is {raw_app_limit_mb}MB (adjustable in instance settings)")))? { let name = field .name() .ok_or_else(|| Error::BadRequest("multipart field missing name".to_string()))? .to_string(); let data = field.bytes().await.map_err(|e| { - Error::BadRequest(format!("failed to read multipart stream: {e}")) + Error::BadRequest(format!("failed to read multipart stream: {e}. Could be due to the request size limit for raw app bundles which is {raw_app_limit_mb}MB (adjustable in instance settings)")) })?; if name == "app" { let app = serde_json::from_slice(&data).map_err(to_anyhow)?; diff --git a/backend/windmill-api/src/args.rs b/backend/windmill-api/src/args.rs index 2734be6740..a881fb989d 100644 --- a/backend/windmill-api/src/args.rs +++ b/backend/windmill-api/src/args.rs @@ -451,8 +451,7 @@ where } } -#[axum::async_trait] -impl FromRequest for RawWebhookArgs +impl FromRequest for RawWebhookArgs where S: Send + Sync, { diff --git a/backend/windmill-api/src/audit.rs b/backend/windmill-api/src/audit.rs index 336fd32881..7f81df849c 100644 --- a/backend/windmill-api/src/audit.rs +++ b/backend/windmill-api/src/audit.rs @@ -19,7 +19,7 @@ use crate::db::ApiAuthed; pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_audit)) - .route("/get/:id", get(get_audit)) + .route("/get/{id}", get(get_audit)) } async fn get_audit( diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 277504b481..c8ddea37d2 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -93,22 +93,22 @@ pub fn workspaced_service() -> Router { Router::new() .route("/set_config", post(set_config)) .route( - "/ping_config/:trigger_kind/:runnable_kind/*path", + "/ping_config/{trigger_kind}/{runnable_kind}/{*path}", post(ping_config), ) - .route("/get_configs/:runnable_kind/*path", get(get_configs)) - .route("/list/:runnable_kind/*path", get(list_captures)) + .route("/get_configs/{runnable_kind}/{*path}", get(get_configs)) + .route("/list/{runnable_kind}/{*path}", get(list_captures)) .route( - "/move/:runnable_kind/*path", + "/move/{runnable_kind}/{*path}", post(move_captures_and_configs), ) - .route("/:id", delete(delete_capture)) - .route("/:id", get(get_capture)) + .route("/{id}", delete(delete_capture)) + .route("/{id}", get(get_capture)) } pub fn workspaced_unauthed_service() -> Router { let router = Router::new().route( - "/webhook/:runnable_kind/*path", + "/webhook/{runnable_kind}/{*path}", head(|| async {}).post(webhook_payload), ); @@ -118,12 +118,12 @@ pub fn workspaced_unauthed_service() -> Router { ))] { #[cfg(feature = "http_trigger")] - let router = router.route("/http/:runnable_kind/:path/*route_path", { + let router = router.route("/http/{runnable_kind}/{path}/{*route_path}", { head(|| async {}).fallback(http_payload) }); #[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] - let router = router.route("/gcp/:runnable_kind/*path", post(gcp_payload)); + let router = router.route("/gcp/{runnable_kind}/{*path}", post(gcp_payload)); router } diff --git a/backend/windmill-api/src/db_health.rs b/backend/windmill-api/src/db_health.rs new file mode 100644 index 0000000000..5b881331c0 --- /dev/null +++ b/backend/windmill-api/src/db_health.rs @@ -0,0 +1,551 @@ +/* + * Author: Windmill Labs + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::{extract::Query, routing::get, Extension, Json, Router}; +use serde::{Deserialize, Serialize}; + +use windmill_common::error::JsonResult; + +use crate::db::{ApiAuthed, DB}; +use crate::utils::require_super_admin; + +pub fn global_service() -> Router { + Router::new().route("/", get(get_db_health)) +} + +// --- Response types --- + +#[derive(Serialize)] +#[serde(rename_all = "lowercase")] +pub enum HealthLevel { + Green, + Yellow, + Red, +} + +#[derive(Serialize)] +pub struct DbHealthResponse { + pub database_size: DatabaseSizeInfo, + pub job_retention: JobRetentionInfo, + pub large_results: LargeResultsInfo, + pub connection_pool: ConnectionPoolInfo, + pub table_maintenance: Vec, + pub slow_queries: Option, + pub datatables: Vec, +} + +#[derive(Serialize)] +pub struct DatabaseSizeInfo { + pub total_size_bytes: i64, + pub total_size_pretty: String, + pub top_tables: Vec, +} + +#[derive(Serialize)] +pub struct TableSizeInfo { + pub table_name: String, + pub total_size_bytes: i64, + pub total_size_pretty: String, +} + +#[derive(Serialize)] +pub struct JobRetentionInfo { + pub oldest_completed_at: Option>, + pub total_completed_jobs: i64, + pub retention_period_secs: Option, + pub status: HealthLevel, + pub message: String, +} + +#[derive(Serialize)] +pub struct LargeResultsInfo { + pub top_large_results: Vec, + pub avg_result_size_bytes: Option, +} + +#[derive(Serialize)] +pub struct LargeResultRow { + pub id: uuid::Uuid, + pub workspace_id: String, + pub runnable_path: Option, + pub result_size_bytes: i64, + pub completed_at: chrono::DateTime, +} + +#[derive(Serialize)] +pub struct ConnectionPoolInfo { + pub pg_max_connections: i64, + pub pg_total_connections: i64, + pub pg_active_connections: i64, + pub pg_idle_connections: i64, + pub status: HealthLevel, + pub message: String, +} + +#[derive(Serialize)] +pub struct TableMaintenanceInfo { + pub table_name: String, + pub live_tuples: i64, + pub dead_tuples: i64, + pub dead_ratio: f64, + pub last_autovacuum: Option, + pub last_autoanalyze: Option, + pub status: HealthLevel, +} + +#[derive(Serialize)] +pub struct SlowQueriesInfo { + pub queries: Vec, + pub message: Option, +} + +#[derive(Serialize)] +pub struct SlowQueryRow { + pub query: String, + pub calls: i64, + pub total_exec_time_ms: f64, + pub mean_exec_time_ms: f64, +} + +#[derive(Serialize)] +pub struct DatatableInfo { + pub workspace_id: String, + pub name: String, + pub table_name: String, + pub size_bytes: i64, + pub size_pretty: String, + pub estimated_rows: f64, +} + +// --- Handler --- + +#[derive(Deserialize)] +struct DbHealthQuery { + /// Max number of recent completed jobs to scan for large results (default 10000) + scan_limit: Option, +} + +async fn get_db_health( + ApiAuthed { email, .. }: ApiAuthed, + Extension(db): Extension, + Query(query): Query, +) -> JsonResult { + require_super_admin(&db, &email).await?; + + let scan_limit = query.scan_limit.unwrap_or(10_000).clamp(1_000, 1_000_000); + + let ( + database_size, + job_retention, + large_results, + connection_pool, + table_maintenance, + slow_queries, + datatables, + ) = tokio::try_join!( + fetch_database_size(&db), + fetch_job_retention(&db), + fetch_large_results(&db, scan_limit), + fetch_connection_pool(&db), + fetch_table_maintenance(&db), + fetch_slow_queries(&db), + fetch_datatables(&db), + )?; + + Ok(Json(DbHealthResponse { + database_size, + job_retention, + large_results, + connection_pool, + table_maintenance, + slow_queries, + datatables, + })) +} + +// --- Diagnostic queries --- + +async fn fetch_database_size(db: &DB) -> windmill_common::error::Result { + let row = sqlx::query!( + "SELECT pg_database_size(current_database()) as size_bytes, pg_size_pretty(pg_database_size(current_database())) as size_pretty" + ) + .fetch_one(db) + .await?; + + let top_tables = sqlx::query_as!( + TableSizeInfo, + r#"SELECT + schemaname || '.' || relname as "table_name!", + pg_total_relation_size(relid) as "total_size_bytes!", + pg_size_pretty(pg_total_relation_size(relid)) as "total_size_pretty!" + FROM pg_catalog.pg_statio_user_tables + ORDER BY pg_total_relation_size(relid) DESC + LIMIT 15"# + ) + .fetch_all(db) + .await?; + + Ok(DatabaseSizeInfo { + total_size_bytes: row.size_bytes.unwrap_or(0), + total_size_pretty: row.size_pretty.unwrap_or_default(), + top_tables, + }) +} + +async fn fetch_job_retention(db: &DB) -> windmill_common::error::Result { + let job_row = + sqlx::query!("SELECT MIN(completed_at) as oldest, COUNT(*) as total FROM v2_job_completed") + .fetch_one(db) + .await?; + + let retention_row = + sqlx::query!("SELECT value FROM global_settings WHERE name = 'retention_period_secs'") + .fetch_optional(db) + .await?; + + let retention_period_secs: Option = + retention_row.map(|r| r.value).and_then(|v| v.as_i64()); + + let oldest = job_row.oldest; + let total = job_row.total.unwrap_or(0); + + let (status, message) = if let (Some(oldest_ts), Some(retention_secs)) = + (oldest, retention_period_secs) + { + let age_secs: i64 = (chrono::Utc::now() - oldest_ts).num_seconds(); + let ratio = if retention_secs > 0 { + age_secs as f64 / retention_secs as f64 + } else { + 0.0 + }; + if ratio <= 2.0 { + ( + HealthLevel::Green, + format!( + "Oldest job is {:.1}x the retention period. Cleanup is keeping up.", + ratio + ), + ) + } else if ratio <= 5.0 { + ( + HealthLevel::Yellow, + format!( + "Oldest job is {:.1}x the retention period. Cleanup may be falling behind.", + ratio + ), + ) + } else { + (HealthLevel::Red, format!("Oldest job is {:.1}x the retention period. Consider reducing retention or investigating cleanup.", ratio)) + } + } else if oldest.is_some() && retention_period_secs.is_none() { + ( + HealthLevel::Yellow, + "No retention_period_secs configured. Old jobs will accumulate.".to_string(), + ) + } else { + (HealthLevel::Green, "No completed jobs found.".to_string()) + }; + + Ok(JobRetentionInfo { + oldest_completed_at: oldest, + total_completed_jobs: total, + retention_period_secs, + status, + message, + }) +} + +async fn fetch_large_results( + db: &DB, + scan_limit: i64, +) -> windmill_common::error::Result { + let top_large_results = sqlx::query_as!( + LargeResultRow, + r#"SELECT + c.id as "id!", + c.workspace_id as "workspace_id!", + j.runnable_path as "runnable_path", + pg_column_size(c.result) as "result_size_bytes!", + c.completed_at as "completed_at!" + FROM ( + SELECT id, workspace_id, result, completed_at + FROM v2_job_completed + WHERE completed_at > now() - interval '30 days' + AND result IS NOT NULL + ORDER BY completed_at DESC + LIMIT $1 + ) c + LEFT JOIN v2_job j ON j.id = c.id + WHERE pg_column_size(c.result) > 1024 + ORDER BY pg_column_size(c.result) DESC + LIMIT 10"#, + scan_limit + ) + .fetch_all(db) + .await?; + + let avg_row = sqlx::query!( + r#"SELECT AVG(pg_column_size(result))::bigint as "avg_size" + FROM ( + SELECT result FROM v2_job_completed + WHERE completed_at > now() - interval '30 days' + AND result IS NOT NULL + ORDER BY completed_at DESC + LIMIT $1 + ) sub"#, + scan_limit + ) + .fetch_one(db) + .await?; + + Ok(LargeResultsInfo { top_large_results, avg_result_size_bytes: avg_row.avg_size }) +} + +async fn fetch_connection_pool(db: &DB) -> windmill_common::error::Result { + let max_row = sqlx::query_scalar!( + r#"SELECT setting::bigint as "max!" FROM pg_settings WHERE name = 'max_connections'"# + ) + .fetch_one(db) + .await?; + + let stats_row = sqlx::query!( + r#"SELECT + COUNT(*) as "total!", + COUNT(*) FILTER (WHERE state = 'active') as "active!", + COUNT(*) FILTER (WHERE state = 'idle') as "idle!" + FROM pg_stat_activity + WHERE backend_type = 'client backend'"# + ) + .fetch_one(db) + .await?; + + let pg_max = max_row; + let pg_total = stats_row.total; + let pg_active = stats_row.active; + let pg_idle = stats_row.idle; + + let utilization = if pg_max > 0 { + pg_total as f64 / pg_max as f64 + } else { + 0.0 + }; + + let (status, message) = if utilization < 0.8 { + ( + HealthLevel::Green, + format!( + "Connection utilization: {:.0}% ({}/{})", + utilization * 100.0, + pg_total, + pg_max + ), + ) + } else if utilization < 0.95 { + ( + HealthLevel::Yellow, + format!( + "Connection utilization is high: {:.0}% ({}/{}). Consider increasing max_connections.", + utilization * 100.0, + pg_total, + pg_max + ), + ) + } else { + ( + HealthLevel::Red, + format!( + "Connections near exhaustion: {:.0}% ({}/{}). Increase max_connections urgently.", + utilization * 100.0, + pg_total, + pg_max + ), + ) + }; + + Ok(ConnectionPoolInfo { + pg_max_connections: pg_max, + pg_total_connections: pg_total, + pg_active_connections: pg_active, + pg_idle_connections: pg_idle, + status, + message, + }) +} + +async fn fetch_table_maintenance( + db: &DB, +) -> windmill_common::error::Result> { + // Aggregate partitioned tables (e.g. audit_YYYYMMDD -> audit_partitioned) + // while keeping non-partitioned tables as-is + let rows = sqlx::query!( + r#"SELECT + table_name as "table_name!", + SUM(live_tuples)::bigint as "live_tuples!", + SUM(dead_tuples)::bigint as "dead_tuples!", + MAX(last_autovacuum) as "last_autovacuum", + MAX(last_autoanalyze) as "last_autoanalyze" + FROM ( + SELECT + CASE + WHEN i.inhparent IS NOT NULL THEN schemaname || '.' || p.relname + ELSE schemaname || '.' || s.relname + END as table_name, + COALESCE(n_live_tup, 0) as live_tuples, + COALESCE(n_dead_tup, 0) as dead_tuples, + last_autovacuum, + last_autoanalyze + FROM pg_stat_user_tables s + LEFT JOIN pg_class c ON c.relname = s.relname AND c.relnamespace = ( + SELECT oid FROM pg_namespace WHERE nspname = s.schemaname + ) + LEFT JOIN pg_inherits i ON i.inhrelid = c.oid + LEFT JOIN pg_class p ON p.oid = i.inhparent + ) sub + GROUP BY table_name + ORDER BY SUM(dead_tuples) DESC"# + ) + .fetch_all(db) + .await?; + + Ok(rows + .into_iter() + .map(|r| { + let total = r.live_tuples + r.dead_tuples; + let dead_ratio = if total > 0 { + r.dead_tuples as f64 / total as f64 + } else { + 0.0 + }; + let status = if dead_ratio < 0.1 { + HealthLevel::Green + } else if dead_ratio < 0.3 { + HealthLevel::Yellow + } else { + HealthLevel::Red + }; + TableMaintenanceInfo { + table_name: r.table_name, + live_tuples: r.live_tuples, + dead_tuples: r.dead_tuples, + dead_ratio, + last_autovacuum: r.last_autovacuum.map(|t| t.naive_utc()), + last_autoanalyze: r.last_autoanalyze.map(|t| t.naive_utc()), + status, + } + }) + .collect()) +} + +async fn fetch_slow_queries(db: &DB) -> windmill_common::error::Result> { + let ext_exists: bool = sqlx::query_scalar!( + r#"SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements') as "exists!""# + ) + .fetch_one(db) + .await?; + + if !ext_exists { + return Ok(Some(SlowQueriesInfo { + queries: vec![], + message: Some( + "pg_stat_statements extension is not installed. Enable it for slow query insights." + .to_string(), + ), + })); + } + + // Use raw query since pg_stat_statements may not exist at compile time + let rows: Vec = sqlx::query_as::<_, (String, i64, f64, f64)>( + r#"SELECT + LEFT(query, 200), + calls::bigint, + total_exec_time::float8, + mean_exec_time::float8 + FROM pg_stat_statements + WHERE query NOT LIKE '%pg_stat_statements%' + ORDER BY mean_exec_time DESC + LIMIT 10"#, + ) + .fetch_all(db) + .await? + .into_iter() + .map( + |(query, calls, total_exec_time_ms, mean_exec_time_ms)| SlowQueryRow { + query, + calls, + total_exec_time_ms, + mean_exec_time_ms, + }, + ) + .collect(); + + Ok(Some(SlowQueriesInfo { queries: rows, message: None })) +} + +async fn fetch_datatables(db: &DB) -> windmill_common::error::Result> { + // Find instance-type datatables from workspace_settings + let rows = sqlx::query!( + r#"SELECT + ws.workspace_id as "workspace_id!", + dt.key as "name!", + dt.value->>'table_name' as "table_name" + FROM workspace_settings ws, + jsonb_each(ws.datatable) dt + WHERE dt.value->>'resource_type' = 'instance' + AND dt.value->>'table_name' IS NOT NULL"# + ) + .fetch_all(db) + .await?; + + let table_names: Vec = rows.iter().filter_map(|r| r.table_name.clone()).collect(); + + if table_names.is_empty() { + return Ok(vec![]); + } + + // Batch lookup: single query for all table sizes + let size_rows = sqlx::query!( + r#"SELECT + c.relname as "table_name!", + pg_total_relation_size(c.oid) as "size_bytes!", + pg_size_pretty(pg_total_relation_size(c.oid)) as "size_pretty!", + COALESCE(c.reltuples, 0) as "estimated_rows!" + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relname = ANY($1)"#, + &table_names + ) + .fetch_all(db) + .await?; + + let size_map: std::collections::HashMap = size_rows + .into_iter() + .map(|s| (s.table_name.clone(), s)) + .collect(); + + let mut result = Vec::new(); + for row in rows { + let table_name = match &row.table_name { + Some(t) => t.clone(), + None => continue, + }; + if let Some(s) = size_map.get(&table_name) { + result.push(DatatableInfo { + workspace_id: row.workspace_id, + name: row.name, + table_name, + size_bytes: s.size_bytes, + size_pretty: s.size_pretty.clone(), + estimated_rows: s.estimated_rows as f64, + }); + } + } + + // Sort by size descending + result.sort_by(|a, b| b.size_bytes.cmp(&a.size_bytes)); + Ok(result) +} diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index 41e8d4709d..39a68d8f9a 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -23,7 +23,7 @@ use windmill_common::{db::UserDB, error::Result, utils::StripPath}; pub fn workspaced_service() -> Router { Router::new() .route("/create", post(create_draft)) - .route("/delete/:kind/*path", delete(delete_draft)) + .route("/delete/{kind}/{*path}", delete(delete_draft)) } #[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)] diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index c6a09a9e6a..15c9262967 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -21,7 +21,7 @@ use windmill_common::{error::JsonResult, utils::StripPath, DB}; /// that depends on windmill-api internals. pub fn workspaced_service() -> Router { windmill_api_flows::flows::workspaced_service() - .route("/get_triggers_count/*path", get(get_triggers_count)) + .route("/get_triggers_count/{*path}", get(get_triggers_count)) } async fn get_triggers_count( diff --git a/backend/windmill-api/src/google.rs b/backend/windmill-api/src/google.rs index dd2a47bca3..fb67f18931 100644 --- a/backend/windmill-api/src/google.rs +++ b/backend/windmill-api/src/google.rs @@ -111,17 +111,16 @@ pub async fn handle_google_ai_chat( let (contents, system_instruction) = openai_messages_to_gemini(&request.messages); - let generation_config = - if request.temperature.is_some() || request.max_tokens.is_some() { - Some(GeminiGenerationConfig { - temperature: request.temperature, - max_output_tokens: request.max_tokens, - response_mime_type: None, - response_schema: None, - }) - } else { - None - }; + let generation_config = if request.temperature.is_some() || request.max_tokens.is_some() { + Some(GeminiGenerationConfig { + temperature: request.temperature, + max_output_tokens: request.max_tokens, + response_mime_type: None, + response_schema: None, + }) + } else { + None + }; let gemini_tools = request.tools.as_ref().map(|tools| { let declarations: Vec = tools @@ -136,10 +135,7 @@ pub async fn handle_google_ai_chat( } }) .collect(); - vec![GeminiTool { - function_declarations: Some(declarations), - google_search: None, - }] + vec![GeminiTool { function_declarations: Some(declarations), google_search: None }] }); let gemini_request = GeminiTextRequest { @@ -184,9 +180,10 @@ async fn handle_streaming( .body(request_body); let request = set_auth(request, api_key, is_vertex); - let response = request.send().await.map_err(|e| { - Error::internal_err(format!("Failed to send request to Gemini API: {}", e)) - })?; + let response = request + .send() + .await + .map_err(|e| Error::internal_err(format!("Failed to send request to Gemini API: {}", e)))?; if let Err(e) = response.error_for_status_ref() { let status = e.status().map(|s| s.to_string()).unwrap_or_default(); @@ -273,9 +270,10 @@ pub async fn handle_google_ai_models( let request = HTTP_CLIENT.get(&endpoint); let request = set_auth(request, api_key, is_vertex); - let response = request.send().await.map_err(|e| { - Error::internal_err(format!("Failed to fetch Gemini models: {}", e)) - })?; + let response = request + .send() + .await + .map_err(|e| Error::internal_err(format!("Failed to fetch Gemini models: {}", e)))?; if let Err(e) = response.error_for_status_ref() { let status = e.status().map(|s| s.to_string()).unwrap_or_default(); @@ -327,9 +325,10 @@ async fn handle_non_streaming( .body(request_body); let request = set_auth(request, api_key, is_vertex); - let response = request.send().await.map_err(|e| { - Error::internal_err(format!("Failed to send request to Gemini API: {}", e)) - })?; + let response = request + .send() + .await + .map_err(|e| Error::internal_err(format!("Failed to send request to Gemini API: {}", e)))?; if let Err(e) = response.error_for_status_ref() { let status = e.status().map(|s| s.to_string()).unwrap_or_default(); @@ -337,9 +336,10 @@ async fn handle_non_streaming( return Err(Error::AIError(format!("{}: {}", status, body))); } - let body = response.bytes().await.map_err(|e| { - Error::internal_err(format!("Failed to read Gemini response body: {}", e)) - })?; + let body = response + .bytes() + .await + .map_err(|e| Error::internal_err(format!("Failed to read Gemini response body: {}", e)))?; let parsed = parse_gemini_response(&body)?; let openai_response = gemini_response_to_openai(&parsed, model); diff --git a/backend/windmill-api/src/group_history.rs b/backend/windmill-api/src/group_history.rs index 0c2c84038d..73162345bc 100644 --- a/backend/windmill-api/src/group_history.rs +++ b/backend/windmill-api/src/group_history.rs @@ -22,7 +22,7 @@ use serde::Serialize; use sqlx::FromRow; pub fn workspaced_service() -> Router { - Router::new().route("/get/:name", get(get_group_permission_history)) + Router::new().route("/get/{name}", get(get_group_permission_history)) } #[derive(Serialize, FromRow)] diff --git a/backend/windmill-api/src/health.rs b/backend/windmill-api/src/health.rs index c31b104429..fab9979c12 100644 --- a/backend/windmill-api/src/health.rs +++ b/backend/windmill-api/src/health.rs @@ -231,7 +231,7 @@ async fn check_database_with_latency(db: &DB) -> DatabaseCheckResult { DatabaseCheckResult { healthy, latency_ms } } -fn get_pool_stats(db: &DB) -> PoolStats { +pub(crate) fn get_pool_stats(db: &DB) -> PoolStats { PoolStats { size: db.size(), idle: db.num_idle() as u32, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index bde3b81a14..8dcbd2a7f4 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -134,14 +134,14 @@ pub fn workspaced_service() -> Router { Router::new() .route( - "/run/f/*script_path", + "/run/f/{*script_path}", post(run_flow_by_path) .head(|| async { "" }) .layer(cors.clone()) .layer(ce_headers.clone()), ) .route( - "/run/fv/:version", + "/run/fv/{version}", post(run_flow_by_version) .head(|| async { "" }) .layer(cors.clone()) @@ -155,25 +155,25 @@ pub fn workspaced_service() -> Router { .layer(ce_headers.clone()), ) .route( - "/run/workflow_as_code/:job_id/:entrypoint", + "/run/workflow_as_code/{job_id}/{entrypoint}", post(run_workflow_as_code) .head(|| async { "" }) .layer(cors.clone()) .layer(ce_headers.clone()), ) .route( - "/restart/f/:job_id", + "/restart/f/{job_id}", post(restart_flow).head(|| async { "" }).layer(cors.clone()), ) .route( - "/run/p/*script_path", + "/run/p/{*script_path}", post(run_script_by_path) .head(|| async { "" }) .layer(cors.clone()) .layer(ce_headers.clone()), ) .route( - "/run_wait_result/p/*script_path", + "/run_wait_result/p/{*script_path}", post(run_wait_result_script_by_path) .get(run_wait_result_job_by_path_get) .head(|| async { "" }) @@ -181,14 +181,14 @@ pub fn workspaced_service() -> Router { .layer(ce_headers.clone()), ) .route( - "/run_wait_result/h/:hash", + "/run_wait_result/h/{hash}", post(run_wait_result_script_by_hash) .head(|| async { "" }) .layer(cors.clone()) .layer(ce_headers.clone()), ) .route( - "/run_wait_result/f/*script_path", + "/run_wait_result/f/{*script_path}", post(run_wait_result_flow_by_path) .get(run_wait_result_flow_by_path_get) .head(|| async { "" }) @@ -196,7 +196,7 @@ pub fn workspaced_service() -> Router { .layer(ce_headers.clone()), ) .route( - "/run_wait_result/fv/:version", + "/run_wait_result/fv/{version}", post(run_wait_result_flow_by_version) .get(run_wait_result_flow_by_version_get) .head(|| async { "" }) @@ -204,7 +204,7 @@ pub fn workspaced_service() -> Router { .layer(ce_headers.clone()), ) .route( - "/run_and_stream/f/*script_path", + "/run_and_stream/f/{*script_path}", get(stream_flow_by_path) .post(stream_flow_by_path) .head(|| async { "" }) @@ -212,7 +212,7 @@ pub fn workspaced_service() -> Router { .layer(ce_headers.clone()), ) .route( - "/run_and_stream/fv/:version", + "/run_and_stream/fv/{version}", get(stream_flow_by_version) .post(stream_flow_by_version) .head(|| async { "" }) @@ -220,7 +220,7 @@ pub fn workspaced_service() -> Router { .layer(ce_headers.clone()), ) .route( - "/run_and_stream/p/*script_path", + "/run_and_stream/p/{*script_path}", get(stream_script_by_path) .post(stream_script_by_path) .head(|| async { "" }) @@ -228,7 +228,7 @@ pub fn workspaced_service() -> Router { .layer(ce_headers.clone()), ) .route( - "/run_and_stream/h/:hash", + "/run_and_stream/h/{hash}", get(stream_script_by_hash) .post(stream_script_by_hash) .head(|| async { "" }) @@ -236,7 +236,7 @@ pub fn workspaced_service() -> Router { .layer(ce_headers.clone()), ) .route( - "/run/h/:hash", + "/run/h/{hash}", post(run_job_by_hash) .head(|| async { "" }) .layer(cors.clone()) @@ -245,10 +245,10 @@ pub fn workspaced_service() -> Router { .route("/run/preview", post(run_preview_script)) .route("/run_inline/preview", post(run_inline_preview_script)) .route( - "/run_inline/p/*script_path", + "/run_inline/p/{*script_path}", post(run_inline_script_by_path), ) - .route("/run_inline/h/:hash", post(run_inline_script_by_hash)) + .route("/run_inline/h/{hash}", post(run_inline_script_by_hash)) .route( "/run_wait_result/preview", post(run_wait_result_preview_script), @@ -257,7 +257,7 @@ pub fn workspaced_service() -> Router { "/run/preview_bundle", post(run_bundle_preview_script).layer(axum::extract::DefaultBodyLimit::disable()), ) - .route("/add_batch_jobs/:n", post(add_batch_jobs)) + .route("/add_batch_jobs/{n}", post(add_batch_jobs)) .route("/run/preview_flow", post(run_preview_flow_job)) .route( "/run_wait_result/preview_flow", @@ -280,8 +280,8 @@ pub fn workspaced_service() -> Router { ) .route("/queue/count", get(count_queue_jobs)) .route("/queue/list_filtered_uuids", get(list_filtered_uuids)) - .route("/queue/position/:timestamp", get(get_queue_position)) - .route("/queue/scheduled_for/:id", get(get_scheduled_for)) + .route("/queue/position/{timestamp}", get(get_queue_position)) + .route("/queue/scheduled_for/{id}", get(get_scheduled_for)) .route("/queue/cancel_selection", post(cancel_selection)) .route("/completed/count", get(count_completed_jobs)) .route("/completed/count_jobs", get(count_completed_jobs_detail)) @@ -299,49 +299,49 @@ pub fn workspaced_service() -> Router { ) .route("/delete", post(crate::jobs_export::delete_jobs)) .route( - "/completed/get/:id", + "/completed/get/{id}", get(get_completed_job).layer(cors.clone()), ) .route( - "/completed/get_result/:id", + "/completed/get_result/{id}", get(get_completed_job_result).layer(cors.clone()), ) .route( - "/completed/get_result_maybe/:id", + "/completed/get_result_maybe/{id}", get(get_completed_job_result_maybe).layer(cors.clone()), ) .route( - "/completed/get_timing/:id", + "/completed/get_timing/{id}", get(get_completed_job_timing).layer(cors.clone()), ) .route( - "/completed/delete/:id", + "/completed/delete/{id}", post(delete_completed_job).layer(cors.clone()), ) .route( - "/flow/resume/:id", + "/flow/resume/{id}", post(resume_suspended_flow_as_owner).layer(cors.clone()), ) .route( - "/job_signature/:job_id/:resume_id", + "/job_signature/{job_id}/{resume_id}", get(create_job_signature).layer(cors.clone()), ) .route( - "/flow/user_states/:job_id/:key", + "/flow/user_states/{job_id}/{key}", get(get_flow_user_state) .post(set_flow_user_state) .layer(cors.clone()), ) .route( - "/resume_urls/:job_id/:resume_id", + "/resume_urls/{job_id}/{resume_id}", get(get_resume_urls).layer(cors.clone()), ) .route( - "/result_by_id/:job_id/:node_id", + "/result_by_id/{job_id}/{node_id}", get(get_result_by_id).layer(cors.clone()), ) .route( - "/flow_env_by_flow_job_id/:flow_job_id/:var_name", + "/flow_env_by_flow_job_id/{flow_job_id}/{var_name}", get(get_flow_env_by_flow_job_id).layer(cors.clone()), ) .route("/run/dependencies", post(run_dependencies_job)) @@ -350,59 +350,59 @@ pub fn workspaced_service() -> Router { "/send_email_with_instance_smtp", post(send_email_with_instance_smtp), ) - .route("/get_otel_traces/:id", get(get_otel_traces)) + .route("/get_otel_traces/{id}", get(get_otel_traces)) } pub fn workspace_unauthed_service() -> Router { Router::new() .route( - "/resume/:job_id/:resume_id/:secret", + "/resume/{job_id}/{resume_id}/{secret}", get(resume_suspended_job), ) .route( - "/resume/:job_id/:resume_id/:secret", + "/resume/{job_id}/{resume_id}/{secret}", post(resume_suspended_job), ) .route( - "/cancel/:job_id/:resume_id/:secret", + "/cancel/{job_id}/{resume_id}/{secret}", get(cancel_suspended_job), ) .route( - "/cancel/:job_id/:resume_id/:secret", + "/cancel/{job_id}/{resume_id}/{secret}", post(cancel_suspended_job), ) .route( - "/get_flow/:job_id/:resume_id/:secret", + "/get_flow/{job_id}/{resume_id}/{secret}", get(get_suspended_job_flow), ) - .route("/get_root_job_id/:id", get(get_root_job)) - .route("/get/:id", get(get_job)) - .route("/get_logs/:id", get(get_job_logs)) + .route("/get_root_job_id/{id}", get(get_root_job)) + .route("/get/{id}", get(get_job)) + .route("/get_logs/{id}", get(get_job_logs)) .route( - "/get_completed_logs_tail/:id", + "/get_completed_logs_tail/{id}", get(get_completed_job_logs_tail), ) - .route("/get_args/:id", get(get_args)) + .route("/get_args/{id}", get(get_args)) .route("/queue/get_started_at_by_ids", post(get_started_at_by_ids)) - .route("/get_flow_debug_info/:id", get(get_flow_job_debug_info)) - .route("/completed/get/:id", get(get_completed_job)) - .route("/completed/get_result/:id", get(get_completed_job_result)) + .route("/get_flow_debug_info/{id}", get(get_flow_job_debug_info)) + .route("/completed/get/{id}", get(get_completed_job)) + .route("/completed/get_result/{id}", get(get_completed_job_result)) .route( - "/completed/get_result_maybe/:id", + "/completed/get_result_maybe/{id}", get(get_completed_job_result_maybe), ) - .route("/completed/get_timing/:id", get(get_completed_job_timing)) - .route("/getupdate/:id", get(get_job_update)) - .route("/getupdate_sse/:id", get(get_job_update_sse)) - .route("/get_log_file/*file_path", get(get_log_file)) - .route("/queue/cancel/:id", post(cancel_job_api)) + .route("/completed/get_timing/{id}", get(get_completed_job_timing)) + .route("/getupdate/{id}", get(get_job_update)) + .route("/getupdate_sse/{id}", get(get_job_update_sse)) + .route("/get_log_file/{*file_path}", get(get_log_file)) + .route("/queue/cancel/{id}", post(cancel_job_api)) .route( - "/queue/cancel_persistent/*script_path", + "/queue/cancel_persistent/{*script_path}", post(cancel_persistent_script_api), ) - .route("/queue/force_cancel/:id", post(force_cancel)) - .route("/flow/resume_suspended/:job_id", post(resume_suspended)) - .route("/flow/approval_info/:job_id", get(get_approval_info)) + .route("/queue/force_cancel/{id}", post(force_cancel)) + .route("/flow/resume_suspended/{job_id}", post(resume_suspended)) + .route("/flow/approval_info/{job_id}", get(get_approval_info)) } pub fn global_root_service() -> Router { @@ -2179,13 +2179,25 @@ async fn list_jobs( pub async fn resume_suspended_flow_as_owner( authed: ApiAuthed, Extension(db): Extension, - Path((_w_id, flow_id)): Path<(String, Uuid)>, + Path((w_id, flow_id)): Path<(String, Uuid)>, QueryOrBody(value): QueryOrBody, ) -> error::Result { let mut tx = db.begin().await?; let (flow, job_id, is_wac) = get_suspended_flow_info(flow_id, &mut tx).await?; + // Verify the job belongs to this workspace + let job_workspace: Option = + sqlx::query_scalar("SELECT workspace_id FROM v2_job WHERE id = $1") + .bind(&flow.id) + .fetch_optional(&mut *tx) + .await?; + if job_workspace.as_deref() != Some(w_id.as_str()) { + return Err(Error::NotFound( + "Job not found in this workspace".to_string(), + )); + } + let flow_path = flow.script_path.as_deref().unwrap_or_else(|| ""); require_owner_of_path(&authed, flow_path)?; check_scopes(&authed, || format!("jobs:run:flows:{}", flow_path))?; @@ -2439,6 +2451,10 @@ struct ApprovalInfo { #[serde(skip_serializing_if = "Option::is_none")] description: Option, #[serde(skip_serializing_if = "Option::is_none")] + default_args: Option, + #[serde(skip_serializing_if = "Option::is_none")] + enums: Option, + #[serde(skip_serializing_if = "Option::is_none")] approval_conditions: Option, can_approve: bool, user_auth_required: bool, @@ -2493,91 +2509,113 @@ async fn get_approval_info( let is_wac = row.workflow_as_code_status.is_some(); // Extract approval info based on WAC vs classic flow - let (form_schema, description, approval_conditions, hide_cancel) = if is_wac { - let approval_meta = row - .workflow_as_code_status - .as_ref() - .and_then(|v| v.get("_approval")); - let form = approval_meta.and_then(|m| m.get("form").cloned()); - let ac = row - .flow_status - .as_ref() - .and_then(|v| v.get("approval_conditions")) - .and_then(|v| serde_json::from_value::(v.clone()).ok()); - (form, None, ac, None) - } else { - let fs = row - .flow_status - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()); - let ac = fs.as_ref().and_then(|s| s.approval_conditions.clone()); + let (form_schema, description, default_args, enums, approval_conditions, hide_cancel) = + if is_wac { + let approval_meta = row + .workflow_as_code_status + .as_ref() + .and_then(|v| v.get("_approval")); + let form = approval_meta.and_then(|m| m.get("form").cloned()); + let default_args = approval_meta.and_then(|m| m.get("default_args").cloned()); + let enums = approval_meta.and_then(|m| m.get("enums").cloned()); + let description = approval_meta.and_then(|m| m.get("description").cloned()); + let ac = row + .flow_status + .as_ref() + .and_then(|v| v.get("approval_conditions")) + .and_then(|v| serde_json::from_value::(v.clone()).ok()); + (form, description, default_args, enums, ac, None) + } else { + let fs = row + .flow_status + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()); + let ac = fs.as_ref().and_then(|s| s.approval_conditions.clone()); - // For classic flows, form/description come from the flow definition and step result - let approval_step = fs.as_ref().map(|s| (s.step as usize).saturating_sub(1)); + // For classic flows, form/description come from the flow definition and step result + let approval_step = fs.as_ref().map(|s| (s.step as usize).saturating_sub(1)); - // Fetch flow definition to get suspend settings (form schema, hide_cancel). - // Try raw_flow on the job first, fall back to flow_version for deployed flows. - let raw_flow: Option = { - let from_job: Option = sqlx::query_scalar( - "SELECT raw_flow FROM v2_job WHERE id = $1 AND workspace_id = $2", - ) - .bind(&job_id) - .bind(&w_id) - .fetch_optional(&db) - .await? - .flatten(); - - if let Some(v) = from_job { - serde_json::from_value(v).ok() - } else { - // Deployed flow: fetch from flow_version using runnable_id - let from_version: Option = sqlx::query_scalar( - "SELECT fv.value FROM v2_job j JOIN flow_version fv ON fv.id = j.runnable_id \ - WHERE j.id = $1 AND j.workspace_id = $2", + // Fetch flow definition to get suspend settings (form schema, hide_cancel). + // Try raw_flow on the job first, fall back to flow_version for deployed flows, + // then flow_node for graph-based branch/loop sub-flows. + let raw_flow: Option = { + let from_job: Option = sqlx::query_scalar( + "SELECT raw_flow FROM v2_job WHERE id = $1 AND workspace_id = $2", ) .bind(&job_id) .bind(&w_id) .fetch_optional(&db) .await? .flatten(); - from_version.and_then(|v| serde_json::from_value(v).ok()) - } + + if let Some(v) = from_job { + serde_json::from_value(v).ok() + } else { + // Deployed flow: fetch from flow_version using runnable_id + let from_version: Option = sqlx::query_scalar( + "SELECT fv.value FROM v2_job j JOIN flow_version fv ON fv.id = j.runnable_id \ + WHERE j.id = $1 AND j.workspace_id = $2", + ) + .bind(&job_id) + .bind(&w_id) + .fetch_optional(&db) + .await? + .flatten(); + if let Some(v) = from_version { + serde_json::from_value(v).ok() + } else { + // FlowNode sub-flow (graph-based branch/loop): raw_flow is not stored + // in v2_job for newer versions, fetch from flow_node table + let from_node: Option = sqlx::query_scalar( + "SELECT fn.flow FROM v2_job j \ + JOIN flow_node fn ON fn.id = j.runnable_id \ + WHERE j.id = $1 AND j.workspace_id = $2", + ) + .bind(&job_id) + .bind(&w_id) + .fetch_optional(&db) + .await? + .flatten(); + from_node.and_then(|v| serde_json::from_value(v).ok()) + } + } + }; + + let suspend_module = raw_flow + .as_ref() + .and_then(|rf| approval_step.and_then(|s| rf.modules.get(s))); + let suspend_settings = suspend_module.and_then(|m| m.suspend.as_ref()); + + let form = suspend_settings + .and_then(|s| s.resume_form.as_ref()) + .map(|rf| serde_json::json!(rf)); + let hc = suspend_settings.map(|s| s.hide_cancel.unwrap_or(false)); + + // Fetch description, default_args, and enums from the step's completed job result + let step_job_id = fs + .as_ref() + .and_then(|s| approval_step.and_then(|step| s.modules.get(step))) + .and_then(|m| m.job()); + let (desc, default_args, enums) = if let Some(sjid) = step_job_id { + let result: Option = sqlx::query_scalar( + "SELECT result FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + ) + .bind(sjid) + .bind(&w_id) + .fetch_optional(&db) + .await? + .flatten(); + let desc = result.as_ref().and_then(|r| r.get("description").cloned()); + let da = result.as_ref().and_then(|r| r.get("default_args").cloned()); + let enums = result.as_ref().and_then(|r| r.get("enums").cloned()); + (desc, da, enums) + } else { + (None, None, None) + }; + + (form, desc, default_args, enums, ac, hc) }; - let suspend_module = raw_flow - .as_ref() - .and_then(|rf| approval_step.and_then(|s| rf.modules.get(s))); - let suspend_settings = suspend_module.and_then(|m| m.suspend.as_ref()); - - let form = suspend_settings - .and_then(|s| s.resume_form.as_ref()) - .map(|rf| serde_json::json!(rf)); - let hc = suspend_settings.map(|s| s.hide_cancel.unwrap_or(false)); - - // Fetch description and default_args from the step's completed job result - let step_job_id = fs - .as_ref() - .and_then(|s| approval_step.and_then(|step| s.modules.get(step))) - .and_then(|m| m.job()); - let (desc, _default_args) = if let Some(sjid) = step_job_id { - let result: Option = sqlx::query_scalar( - "SELECT result FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", - ) - .bind(sjid) - .bind(&w_id) - .fetch_optional(&db) - .await? - .flatten(); - let desc = result.as_ref().and_then(|r| r.get("description").cloned()); - let da = result.as_ref().and_then(|r| r.get("default_args").cloned()); - (desc, da) - } else { - (None, None) - }; - - (form, desc, ac, hc) - }; - let user_auth_required = approval_conditions .as_ref() .map(|ac| ac.user_auth_required) @@ -2628,6 +2666,8 @@ async fn get_approval_info( flow_id: row.id, form_schema, description, + default_args, + enums, approval_conditions, can_approve, user_auth_required, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 15a674ea37..fd356de67f 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -37,6 +37,7 @@ use axum::body::Body; use axum::extract::DefaultBodyLimit; use axum::http::HeaderValue; use axum::response::Response; +use axum::serve::ListenerExt; use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Json, Router}; use db::DB; use tokio::task::JoinHandle; @@ -76,6 +77,7 @@ mod bedrock; mod capture; mod concurrency_groups; mod db; +mod db_health; mod google; mod drafts; @@ -377,6 +379,8 @@ pub async fn run_server( REQUEST_SIZE_LIMIT.read().await.clone(), )); + let request_size_limit = REQUEST_SIZE_LIMIT.read().await.clone(); + let cors = CorsLayer::new() .allow_methods([http::Method::GET, http::Method::POST, http::Method::DELETE]) .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION]) @@ -527,11 +531,11 @@ pub async fn run_server( "/api", Router::new() .nest( - "/w/:workspace_id", + "/w/{workspace_id}", Router::new() // Reordered alphabetically .nest("/acls", granular_acls::workspaced_service()) - .nest("/apps", apps::workspaced_service()) + .nest("/apps", apps::workspaced_service(request_size_limit * 5)) .nest("/assets", windmill_api_assets::workspaced_service()) .nest("/audit", audit::workspaced_service()) .nest("/capture", capture::workspaced_service()) @@ -640,7 +644,8 @@ pub async fn run_server( .nest("/ai", ai::global_service()) .nest("/inkeep", inkeep_oss::global_service()) .nest("/indexer", indexer_oss::management_service()) - .nest("/mcp/w/:workspace_id/list_tools", mcp_list_tools_service) + .nest("/mcp/w/{workspace_id}/list_tools", mcp_list_tools_service) + .nest("/db_health", db_health::global_service()) .nest("/health/detailed", health::detailed_service()) .nest( "/saml", @@ -659,7 +664,7 @@ pub async fn run_server( .route_layer(from_extractor::()) // Workspace-scoped OAuth endpoints that don't require authentication // (authorize and token are called by MCP client before user is authenticated) - .nest("/w/:workspace_id/mcp/oauth/server", { + .nest("/w/{workspace_id}/mcp/oauth/server", { #[cfg(feature = "mcp")] { mcp::oauth_server::workspaced_unauthed_service() @@ -680,7 +685,7 @@ pub async fn run_server( }) .nest("/jobs", jobs::global_root_service()) .nest( - "/srch/w/:workspace_id/index", + "/srch/w/{workspace_id}/index", indexer_oss::workspaced_service(), ) .nest("/srch/index", indexer_oss::global_service()) @@ -710,19 +715,19 @@ pub async fn run_server( } }) .nest( - "/w/:workspace_id/apps_u", + "/w/{workspace_id}/apps_u", apps::unauthed_service() .layer(from_extractor::()) .layer(cors.clone()), ) .layer(from_extractor::()) - // Deprecated, here for backwards compatibility: user should use /mcp/w/:workspace_id/mcp instead + // Deprecated, here for backwards compatibility: user should use /mcp/w/{workspace_id}/mcp instead .nest( - "/mcp/w/:workspace_id/sse", + "/mcp/w/{workspace_id}/sse", mcp_router.clone().layer(cors.clone()), ) .nest( - "/mcp/w/:workspace_id/mcp", + "/mcp/w/{workspace_id}/mcp", mcp_router.clone().layer(cors.clone()), ) .nest("/mcp/gateway", gateway_mcp_router.layer(cors.clone())) @@ -745,7 +750,7 @@ pub async fn run_server( Router::new() } }) - .nest("/w/:workspace_id/agent_workers", { + .nest("/w/{workspace_id}/agent_workers", { #[cfg(feature = "agent_worker_server")] { agent_workers_router @@ -762,7 +767,7 @@ pub async fn run_server( } }) .nest( - "/w/:workspace_id/jobs_u", + "/w/{workspace_id}/jobs_u", jobs::workspace_unauthed_service().layer(cors.clone()), ) .route("/slack", post(slack_approvals::slack_app_callback_handler)) @@ -778,14 +783,14 @@ pub async fn run_server( } }) .route( - "/w/:workspace_id/jobs/slack_approval/:job_id", + "/w/{workspace_id}/jobs/slack_approval/{job_id}", get(slack_approvals::request_slack_approval), ) .route( - "/w/:workspace_id/jobs/teams_approval/:job_id", + "/w/{workspace_id}/jobs/teams_approval/{job_id}", get(teams_approvals_oss::request_teams_approval), ) - .nest("/w/:workspace_id/github_app", { + .nest("/w/{workspace_id}/github_app", { #[cfg(feature = "enterprise")] { git_sync_oss::workspaced_service() @@ -804,14 +809,14 @@ pub async fn run_server( Router::new() }) .nest( - "/w/:workspace_id/resources_u", + "/w/{workspace_id}/resources_u", public_service().layer(cors.clone()), ) .nest( - "/w/:workspace_id/capture_u", + "/w/{workspace_id}/capture_u", capture::workspaced_unauthed_service().layer(cors.clone()), ) - .nest("/w/:workspace_id/s3_proxy", { + .nest("/w/{workspace_id}/s3_proxy", { s3_proxy_oss::workspaced_unauthed_service() }) .nest( @@ -856,7 +861,7 @@ pub async fn run_server( Router::new() } }) - .nest("/gcp/w/:workspace_id", { + .nest("/gcp/w/{workspace_id}", { #[cfg(all( feature = "enterprise", feature = "gcp_trigger", @@ -883,10 +888,10 @@ pub async fn run_server( .route("/openapi.json", get(openapi_json)), ) // Clients must use workspace-scoped OAuth metadata at: - // /.well-known/oauth-authorization-server/api/w/:workspace_id/mcp/oauth/server + // /.well-known/oauth-authorization-server/api/w/{workspace_id}/mcp/oauth/server // This is discovered via /.well-known/oauth-protected-resource?workspace_id=... .route( - "/.well-known/oauth-authorization-server/api/w/:workspace_id/mcp/oauth/server", + "/.well-known/oauth-authorization-server/api/w/{workspace_id}/mcp/oauth/server", { #[cfg(feature = "mcp")] { @@ -898,9 +903,9 @@ pub async fn run_server( } }, ) - // RFC 9728 path-based discovery: /.well-known/oauth-protected-resource/api/mcp/w/:workspace_id/mcp + // RFC 9728 path-based discovery: /.well-known/oauth-protected-resource/api/mcp/w/{workspace_id}/mcp .route( - "/.well-known/oauth-protected-resource/api/mcp/w/:workspace_id/mcp", + "/.well-known/oauth-protected-resource/api/mcp/w/{workspace_id}/mcp", { #[cfg(feature = "mcp")] { @@ -976,7 +981,10 @@ pub async fn run_server( if let Some(name) = name.as_ref() { tracing::info!("server starting for name={name}"); } - let server = axum::serve(listener, app.into_make_service()).tcp_nodelay(!server_mode); + let listener = listener.tap_io(move |tcp_stream| { + let _ = tcp_stream.set_nodelay(!server_mode); + }); + let server = axum::serve(listener, app.into_make_service()); tracing::info!( instance = %*INSTANCE_NAME, diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index 9425b6a01f..c9295ea8e9 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -221,6 +221,22 @@ pub fn all_tools() -> Vec { "type": "string", "description": "filter variables by path prefix" }, + "path": { + "type": "string", + "description": "exact path match filter" + }, + "description": { + "type": "string", + "description": "pattern match filter for description field (case-insensitive)" + }, + "value": { + "type": "string", + "description": "pattern match filter for non-secret variable values (case-insensitive)" + }, + "broad_filter": { + "type": "string", + "description": "broad search across multiple fields (case-insensitive substring match)" + }, "page": { "type": "integer", "description": "which page to return (start at 1, default 1)" @@ -405,6 +421,22 @@ pub fn all_tools() -> Vec { "path_start": { "type": "string", "description": "filter resources by path prefix" + }, + "path": { + "type": "string", + "description": "exact path match filter" + }, + "description": { + "type": "string", + "description": "pattern match filter for description field (case-insensitive)" + }, + "value": { + "type": "string", + "description": "JSONB subset match filter using base64 encoded JSON" + }, + "broad_filter": { + "type": "string", + "description": "broad search across multiple fields (case-insensitive substring match)" } }, "required": [] @@ -451,7 +483,7 @@ pub fn all_tools() -> Vec { }, "created_by": { "type": "string", - "description": "mask to filter exact matching user creator" + "description": "filter by exact matching user creator. Supports comma-separated list (e.g. 'alice,bob') and negation by prefixing all values with '!' (e.g. '!alice,!bob')" }, "path_start": { "type": "string", @@ -562,7 +594,6 @@ pub fn all_tools() -> Vec { "required": [ "path", "summary", - "description", "content", "language" ] @@ -708,7 +739,7 @@ pub fn all_tools() -> Vec { }, "created_by": { "type": "string", - "description": "mask to filter exact matching user creator" + "description": "filter by exact matching user creator. Supports comma-separated list (e.g. 'alice,bob') and negation by prefixing all values with '!' (e.g. '!alice,!bob')" }, "path_start": { "type": "string", @@ -1078,6 +1109,37 @@ pub fn all_tools() -> Vec { }, "lock": { "type": "string" + }, + "flow_path": { + "type": "string" + }, + "modules": { + "type": "object", + "nullable": true, + "description": "Additional script modules keyed by relative file path", + "additionalProperties": { + "type": "object", + "description": "An additional module file associated with a script", + "properties": { + "content": { + "type": "string", + "description": "The source code content of this module" + }, + "language": { + "type": "string", + "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, duckdb, bunnative" + }, + "lock": { + "type": "string", + "nullable": true, + "description": "Lock file content for this module's dependencies" + } + }, + "required": [ + "content", + "language" + ] + } } }, "required": [ @@ -1106,7 +1168,7 @@ pub fn all_tools() -> Vec { }, "created_by": { "type": "string", - "description": "mask to filter exact matching user creator" + "description": "filter by exact matching user creator. Supports comma-separated list (e.g. 'alice,bob') and negation by prefixing all values with '!' (e.g. '!alice,!bob')" }, "parent_job": { "type": "string", @@ -1115,15 +1177,15 @@ pub fn all_tools() -> Vec { }, "worker": { "type": "string", - "description": "worker this job was ran on" + "description": "filter by worker this job ran on. Supports comma-separated list (e.g. 'worker-1,worker-2') and negation by prefixing all values with '!' (e.g. '!worker-1,!worker-2')" }, "script_path_exact": { "type": "string", - "description": "mask to filter exact matching path" + "description": "filter by exact matching script path. Supports comma-separated list (e.g. 'f/script1,f/script2') and negation by prefixing all values with '!' (e.g. '!f/script1,!f/script2')" }, "script_path_start": { "type": "string", - "description": "mask to filter matching starting path" + "description": "filter by script path prefix. Supports comma-separated list (e.g. 'f/folder1,f/folder2') and negation by prefixing all values with '!' (e.g. '!f/folder1,!f/folder2')" }, "schedule_path": { "type": "string", @@ -1131,11 +1193,11 @@ pub fn all_tools() -> Vec { }, "trigger_path": { "type": "string", - "description": "mask to filter by trigger path" + "description": "filter by trigger path. Supports comma-separated list (e.g. 'f/trigger1,f/trigger2') and negation by prefixing all values with '!' (e.g. '!f/trigger1,!f/trigger2')" }, "trigger_kind": { - "description": "trigger kind (schedule, http, websocket...). Possible values: webhook, default_email, email, schedule, http, websocket, postgres, kafka, nats, mqtt, sqs, gcp", - "type": "string" + "type": "string", + "description": "filter by trigger kind. Supports comma-separated list (e.g. 'schedule,webhook') and negation by prefixing all values with '!' (e.g. '!schedule,!webhook')" }, "script_hash": { "type": "string", @@ -1161,7 +1223,7 @@ pub fn all_tools() -> Vec { }, "job_kinds": { "type": "string", - "description": "filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by," + "description": "filter by job kind. Supports comma-separated list of values ('preview', 'script', 'dependencies', 'flow') and negation by prefixing all values with '!' (e.g. '!preview,!dependencies')" }, "suspended": { "type": "boolean", @@ -1185,7 +1247,7 @@ pub fn all_tools() -> Vec { }, "tag": { "type": "string", - "description": "filter on jobs with a given tag/worker group" + "description": "filter by tag/worker group. Supports comma-separated list (e.g. 'gpu,highmem') and negation by prefixing all values with '!' (e.g. '!gpu,!highmem')" }, "page": { "type": "integer", @@ -1223,15 +1285,15 @@ pub fn all_tools() -> Vec { "properties": { "created_by": { "type": "string", - "description": "mask to filter exact matching user creator" + "description": "filter by exact matching user creator. Supports comma-separated list (e.g. 'alice,bob') and negation by prefixing all values with '!' (e.g. '!alice,!bob')" }, "label": { "type": "string", - "description": "mask to filter exact matching job's label (job labels are completed jobs with as a result an object containing a string in the array at key 'wm_labels')" + "description": "filter by exact matching job label. Supports comma-separated list (e.g. 'deploy,release') and negation by prefixing all values with '!' (e.g. '!deploy,!release')" }, "worker": { "type": "string", - "description": "worker this job was ran on" + "description": "filter by worker this job ran on. Supports comma-separated list (e.g. 'worker-1,worker-2') and negation by prefixing all values with '!' (e.g. '!worker-1,!worker-2')" }, "parent_job": { "type": "string", @@ -1240,11 +1302,11 @@ pub fn all_tools() -> Vec { }, "script_path_exact": { "type": "string", - "description": "mask to filter exact matching path" + "description": "filter by exact matching script path. Supports comma-separated list (e.g. 'f/script1,f/script2') and negation by prefixing all values with '!' (e.g. '!f/script1,!f/script2')" }, "script_path_start": { "type": "string", - "description": "mask to filter matching starting path" + "description": "filter by script path prefix. Supports comma-separated list (e.g. 'f/folder1,f/folder2') and negation by prefixing all values with '!' (e.g. '!f/folder1,!f/folder2')" }, "schedule_path": { "type": "string", @@ -1304,7 +1366,7 @@ pub fn all_tools() -> Vec { }, "job_kinds": { "type": "string", - "description": "filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by," + "description": "filter by job kind. Supports comma-separated list of values ('preview', 'script', 'dependencies', 'flow') and negation by prefixing all values with '!' (e.g. '!preview,!dependencies')" }, "suspended": { "type": "boolean", @@ -1316,7 +1378,7 @@ pub fn all_tools() -> Vec { }, "tag": { "type": "string", - "description": "filter on jobs with a given tag/worker group" + "description": "filter by tag/worker group. Supports comma-separated list (e.g. 'gpu,highmem') and negation by prefixing all values with '!' (e.g. '!gpu,!highmem')" }, "result": { "type": "string", @@ -1331,8 +1393,8 @@ pub fn all_tools() -> Vec { "description": "number of items to return for a given page (default 30, max 100)" }, "trigger_kind": { - "description": "trigger kind (schedule, http, websocket...). Possible values: webhook, default_email, email, schedule, http, websocket, postgres, kafka, nats, mqtt, sqs, gcp", - "type": "string" + "type": "string", + "description": "filter by trigger kind. Supports comma-separated list (e.g. 'schedule,webhook') and negation by prefixing all values with '!' (e.g. '!schedule,!webhook')" }, "is_skipped": { "type": "boolean", @@ -1357,6 +1419,77 @@ pub fn all_tools() -> Vec { "is_not_schedule": { "type": "boolean", "description": "is not a scheduled job" + }, + "broad_filter": { + "type": "string", + "description": "broad search across multiple fields (case-insensitive substring match on path, tag, schedule path, trigger kind, label)" + } + }, + "required": [] +})), + body_schema: None, + path_field_renames: None, + query_field_renames: None, + body_field_renames: None, + }, + EndpointTool { + name: Cow::Borrowed("getJob"), + description: Cow::Borrowed("get job"), + instructions: Cow::Borrowed(""), + path: Cow::Borrowed("/w/{workspace}/jobs_u/get/{id}"), + method: Cow::Borrowed("GET"), + path_params_schema: Some(serde_json::json!({ + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "id" + ] +})), + query_params_schema: Some(serde_json::json!({ + "type": "object", + "properties": { + "no_logs": { + "type": "boolean" + }, + "no_code": { + "type": "boolean" + } + }, + "required": [] +})), + body_schema: None, + path_field_renames: None, + query_field_renames: None, + body_field_renames: None, + }, + EndpointTool { + name: Cow::Borrowed("getJobLogs"), + description: Cow::Borrowed("get job logs"), + instructions: Cow::Borrowed(""), + path: Cow::Borrowed("/w/{workspace}/jobs_u/get_logs/{id}"), + method: Cow::Borrowed("GET"), + path_params_schema: Some(serde_json::json!({ + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "id" + ] +})), + query_params_schema: Some(serde_json::json!({ + "type": "object", + "properties": { + "remove_ansi_warnings": { + "type": "boolean" } }, "required": [] @@ -1411,14 +1544,17 @@ You should get the schema of the script or flow before creating the schedule to }, "on_failure": { "type": "string", + "nullable": true, "description": "Path to a script or flow to run when the scheduled job fails" }, "on_failure_times": { "type": "number", + "nullable": true, "description": "Number of consecutive failures before the on_failure handler is triggered (default 1)" }, "on_failure_exact": { "type": "boolean", + "nullable": true, "description": "If true, trigger on_failure handler only on exactly N failures, not on every failure after N" }, "on_failure_extra_args": { @@ -1428,10 +1564,12 @@ You should get the schema of the script or flow before creating the schedule to }, "on_recovery": { "type": "string", + "nullable": true, "description": "Path to a script or flow to run when the schedule recovers after failures" }, "on_recovery_times": { "type": "number", + "nullable": true, "description": "Number of consecutive successes before the on_recovery handler is triggered (default 1)" }, "on_recovery_extra_args": { @@ -1441,6 +1579,7 @@ You should get the schema of the script or flow before creating the schedule to }, "on_success": { "type": "string", + "nullable": true, "description": "Path to a script or flow to run after each successful execution" }, "on_success_extra_args": { @@ -1516,28 +1655,42 @@ You should get the schema of the script or flow before creating the schedule to }, "summary": { "type": "string", + "nullable": true, "description": "Short summary describing the purpose of this schedule" }, "description": { "type": "string", + "nullable": true, "description": "Detailed description of what this schedule does" }, "tag": { "type": "string", + "nullable": true, "description": "Worker tag to route jobs to specific worker groups" }, "paused_until": { "type": "string", + "nullable": true, "format": "date-time", "description": "ISO 8601 datetime until which the schedule is paused. Schedule resumes automatically after this time" }, "cron_version": { "type": "string", + "nullable": true, "description": "Cron parser version. Use 'v2' for extended syntax with additional features" }, "dynamic_skip": { "type": "string", + "nullable": true, "description": "Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean to skip (true) or run (false)" + }, + "permissioned_as": { + "type": "string", + "description": "The user or group this schedule runs as. Used during deployment to preserve the original schedule owner." + }, + "preserve_permissioned_as": { + "type": "boolean", + "description": "When true and the caller is a member of the 'wm_deployers' group, preserves the original permissioned_as value instead of overwriting it." } }, "required": [ @@ -1592,14 +1745,17 @@ You should get the schema of the script or flow before updating the schedule to }, "on_failure": { "type": "string", + "nullable": true, "description": "Path to a script or flow to run when the scheduled job fails" }, "on_failure_times": { "type": "number", + "nullable": true, "description": "Number of consecutive failures before the on_failure handler is triggered (default 1)" }, "on_failure_exact": { "type": "boolean", + "nullable": true, "description": "If true, trigger on_failure handler only on exactly N failures, not on every failure after N" }, "on_failure_extra_args": { @@ -1609,10 +1765,12 @@ You should get the schema of the script or flow before updating the schedule to }, "on_recovery": { "type": "string", + "nullable": true, "description": "Path to a script or flow to run when the schedule recovers after failures" }, "on_recovery_times": { "type": "number", + "nullable": true, "description": "Number of consecutive successes before the on_recovery handler is triggered (default 1)" }, "on_recovery_extra_args": { @@ -1622,6 +1780,7 @@ You should get the schema of the script or flow before updating the schedule to }, "on_success": { "type": "string", + "nullable": true, "description": "Path to a script or flow to run after each successful execution" }, "on_success_extra_args": { @@ -1697,28 +1856,44 @@ You should get the schema of the script or flow before updating the schedule to }, "summary": { "type": "string", + "nullable": true, "description": "Short summary describing the purpose of this schedule" }, "description": { "type": "string", + "nullable": true, "description": "Detailed description of what this schedule does" }, "tag": { "type": "string", + "nullable": true, "description": "Worker tag to route jobs to specific worker groups" }, "paused_until": { "type": "string", + "nullable": true, "format": "date-time", "description": "ISO 8601 datetime until which the schedule is paused. Schedule resumes automatically after this time" }, "cron_version": { "type": "string", + "nullable": true, "description": "Cron parser version. Use 'v2' for extended syntax with additional features" }, "dynamic_skip": { "type": "string", + "nullable": true, "description": "Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean to skip (true) or run (false)" + }, + "permissioned_as": { + "type": "string", + "nullable": true, + "description": "The user or group this schedule runs as (e.g., 'u/admin' or 'g/mygroup'). Only admins and wm_deployers can set this via preserve_permissioned_as." + }, + "preserve_permissioned_as": { + "type": "boolean", + "nullable": true, + "description": "If true and user is admin/wm_deployers, preserve the provided permissioned_as instead of using the deploying user's identity" } }, "required": [ @@ -1801,7 +1976,7 @@ You should get the schema of the script or flow before updating the schedule to }, "path": { "type": "string", - "description": "filter by path" + "description": "filter by path (script path)" }, "is_flow": { "type": "boolean", @@ -1810,6 +1985,22 @@ You should get the schema of the script or flow before updating the schedule to "path_start": { "type": "string", "description": "filter schedules by path prefix" + }, + "schedule_path": { + "type": "string", + "description": "exact match on the schedule's path" + }, + "description": { + "type": "string", + "description": "pattern match filter for description field (case-insensitive)" + }, + "summary": { + "type": "string", + "description": "pattern match filter for summary field (case-insensitive)" + }, + "broad_filter": { + "type": "string", + "description": "broad search across multiple fields (case-insensitive substring match)" } }, "required": [] diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index 7699853559..a20484e3eb 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -546,7 +546,7 @@ pub async fn setup_mcp_server( let service = StreamableHttpService::new(move || Ok(runner.clone()), session_manager, service_config); - let router = Router::new().nest_service("/", service); + let router = Router::new().route_service("/", service); Ok((router, cancellation_token)) } diff --git a/backend/windmill-api/src/raw_apps.rs b/backend/windmill-api/src/raw_apps.rs index e331aa1176..ed746b8770 100644 --- a/backend/windmill-api/src/raw_apps.rs +++ b/backend/windmill-api/src/raw_apps.rs @@ -27,7 +27,7 @@ use windmill_common::{ pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_apps)) - .route("/get_data/:version/*path", get(get_data)) + .route("/get_data/{version}/{*path}", get(get_data)) } #[derive(FromRow, Deserialize, Serialize)] diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index f4bcfc621b..ee26ce758e 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -10,7 +10,7 @@ pub fn workspaced_service() -> Router { #[cfg(feature = "mcp")] use crate::mcp_tools::get_mcp_tools; #[cfg(feature = "mcp")] - let router = router.route("/mcp_tools/*path", get(get_mcp_tools)); + let router = router.route("/mcp_tools/{*path}", get(get_mcp_tools)); router } diff --git a/backend/windmill-api/src/scim_oss.rs b/backend/windmill-api/src/scim_oss.rs index 5210411466..845c854960 100644 --- a/backend/windmill-api/src/scim_oss.rs +++ b/backend/windmill-api/src/scim_oss.rs @@ -11,9 +11,7 @@ pub use crate::scim_ee::*; */ #[cfg(not(feature = "private"))] -use axum::{middleware::Next, response::Response, routing::get, Router}; -#[cfg(not(feature = "private"))] -use hyper::Request; +use axum::{extract::Request, middleware::Next, response::Response, routing::get, Router}; #[cfg(not(feature = "private"))] pub fn global_service() -> Router { @@ -26,7 +24,7 @@ pub async fn ee() -> String { } #[cfg(not(feature = "private"))] -pub async fn has_scim_token(_request: Request, _next: Next) -> Response { +pub async fn has_scim_token(_request: Request, _next: Next) -> Response { //Not implemented in open-source version todo!() } diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index abd0b4201b..f9dd17fa83 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -21,7 +21,7 @@ use windmill_common::{error::JsonResult, utils::StripPath, DB}; /// that depends on windmill-api internals. pub fn workspaced_service() -> Router { windmill_api_scripts::scripts::workspaced_service() - .route("/get_triggers_count/*path", get(get_triggers_count)) + .route("/get_triggers_count/{*path}", get(get_triggers_count)) } async fn get_triggers_count( diff --git a/backend/windmill-api/src/service_logs.rs b/backend/windmill-api/src/service_logs.rs index c83bb21f2c..57131d823e 100644 --- a/backend/windmill-api/src/service_logs.rs +++ b/backend/windmill-api/src/service_logs.rs @@ -20,7 +20,7 @@ use crate::db::{ApiAuthed, DB}; pub fn global_service() -> Router { Router::new() .route("/list_files", get(list_files)) - .route("/get_log_file/*path", get(get_log_file)) + .route("/get_log_file/{*path}", get(get_log_file)) } use axum::extract::Path; @@ -97,6 +97,9 @@ async fn get_log_file( require_devops_role(&db, &email).await?; let path = path.to_path(); + if path.contains("..") { + return Err(Error::BadRequest("Invalid path".to_string())); + } #[cfg(feature = "parquet")] let s3_client = windmill_object_store::get_object_store().await; #[cfg(feature = "parquet")] diff --git a/backend/windmill-api/src/trash.rs b/backend/windmill-api/src/trash.rs index 8e7b7f0ad7..a4bc105238 100644 --- a/backend/windmill-api/src/trash.rs +++ b/backend/windmill-api/src/trash.rs @@ -17,9 +17,9 @@ use crate::db::{ApiAuthed, DB}; pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_trash)) - .route("/get/:id", get(get_trash_item)) - .route("/restore/:id", post(restore_trash_item)) - .route("/delete/:id", delete(permanently_delete_item)) + .route("/get/{id}", get(get_trash_item)) + .route("/restore/{id}", post(restore_trash_item)) + .route("/delete/{id}", delete(permanently_delete_item)) .route("/empty", post(empty_trash)) } diff --git a/backend/windmill-api/src/triggers/handler.rs b/backend/windmill-api/src/triggers/handler.rs index 78715e2661..f798b4d706 100644 --- a/backend/windmill-api/src/triggers/handler.rs +++ b/backend/windmill-api/src/triggers/handler.rs @@ -108,11 +108,11 @@ pub fn generate_trigger_routers() -> Router { router = router .route( - "/trigger/:trigger_kind/resume_suspended_trigger_jobs/*trigger_path", + "/trigger/{trigger_kind}/resume_suspended_trigger_jobs/{*trigger_path}", post(resume_suspended_trigger_jobs), ) .route( - "/trigger/:trigger_kind/cancel_suspended_trigger_jobs/*trigger_path", + "/trigger/{trigger_kind}/cancel_suspended_trigger_jobs/{*trigger_path}", post(cancel_suspended_trigger_jobs), ); } diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index 8af10bda18..ccde55bab1 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -95,7 +95,7 @@ async fn conditional_cors_middleware( pub fn http_route_trigger_handler() -> Router { Router::new() .route( - "/*path", + "/{*path}", get(route_job) .post(route_job) .delete(route_job) diff --git a/backend/windmill-api/src/triggers/http/http_trigger_args.rs b/backend/windmill-api/src/triggers/http/http_trigger_args.rs index 7df1e9f200..f8460f42e2 100644 --- a/backend/windmill-api/src/triggers/http/http_trigger_args.rs +++ b/backend/windmill-api/src/triggers/http/http_trigger_args.rs @@ -25,8 +25,7 @@ use crate::{ pub struct RawHttpTriggerArgs(pub RawWebhookArgs); -#[axum::async_trait] -impl FromRequest for RawHttpTriggerArgs +impl FromRequest for RawHttpTriggerArgs where S: Send + Sync, { diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 498e93e61a..aec6080bfd 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -35,9 +35,9 @@ use windmill_common::{ pub fn global_service() -> Router { windmill_api_users::users::global_service() .route("/setpassword", post(set_password)) - .route("/set_password_of/:user", post(set_password_of_user)) + .route("/set_password_of/{user}", post(set_password_of_user)) .route("/create", post(create_user)) - .route("/rename/:user", post(rename_user)) + .route("/rename/{user}", post(rename_user)) .route("/onboarding", post(submit_onboarding_data)) } diff --git a/backend/windmill-api/src/workspace_dependencies.rs b/backend/windmill-api/src/workspace_dependencies.rs index e5c9377b01..194cc1f0d5 100644 --- a/backend/windmill-api/src/workspace_dependencies.rs +++ b/backend/windmill-api/src/workspace_dependencies.rs @@ -24,9 +24,9 @@ pub fn workspaced_service() -> Router { Router::new() .route("/create", post(create)) .route("/list", get(list)) - .route("/archive/:language", post(archive)) - .route("/get_latest/:language", get(get_latest)) - .route("/delete/:language", post(delete)) + .route("/archive/{language}", post(archive)) + .route("/get_latest/{language}", get(get_latest)) + .route("/delete/{language}", post(delete)) } #[axum::debug_handler] diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 3438ad97af..6d644a7946 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -63,7 +63,7 @@ pub fn workspaced_service() -> Router { .route("/get_copilot_info", get(get_copilot_info)) .route("/critical_alerts", get(get_critical_alerts)) .route( - "/critical_alerts/:id/acknowledge", + "/critical_alerts/{id}/acknowledge", post(acknowledge_critical_alert), ) .route( diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 6c6c783ba7..d6b5e35e98 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -484,6 +484,7 @@ pub(crate) async fn tarball_workspace( ScriptLang::OracleDB => "odb.sql", ScriptLang::Java => "java", ScriptLang::Ruby => "rb", + ScriptLang::Rlang => "r", // for related places search: ADD_NEW_LANG }; archive diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index d593787def..fcf69c13ee 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -117,6 +117,7 @@ pin-project-lite.workspace = true futures.workspace = true tempfile.workspace = true globset.workspace = true +dashmap.workspace = true opentelemetry-semantic-conventions = { workspace = true, optional = true } opentelemetry-otlp = { workspace = true, optional = true } diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index b3186cc6a2..e39fd6ea43 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -1231,3 +1231,75 @@ const _: () = { } } }; + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn flow_data_extras_preserves_notes_and_groups() { + let raw = serde_json::value::to_raw_value(&json!({ + "modules": [], + "notes": [{"id": "n1", "text": "hello", "color": "blue", "type": "group", + "contained_node_ids": ["a", "b"], "locked": false}], + "groups": [{"start_id": "a", "end_id": "b", "summary": "grp", "color": "green"}] + })) + .unwrap(); + + let data = FlowData::from_raw(raw).unwrap(); + + // FlowValue ignores notes/groups + assert!(data.value().modules.is_empty()); + + // But extras() recovers them from the raw JSON + let extras = data.extras().expect("extras should parse"); + let notes: serde_json::Value = + serde_json::from_str(extras.notes.expect("notes present").get()).unwrap(); + assert_eq!(notes.as_array().unwrap().len(), 1); + assert_eq!(notes[0]["id"], "n1"); + assert_eq!(notes[0]["color"], "blue"); + + let groups: serde_json::Value = + serde_json::from_str(extras.groups.expect("groups present").get()).unwrap(); + assert_eq!(groups.as_array().unwrap().len(), 1); + assert_eq!(groups[0]["start_id"], "a"); + } + + #[test] + fn flow_data_extras_returns_none_when_missing() { + let raw = serde_json::value::to_raw_value(&json!({"modules": []})).unwrap(); + let data = FlowData::from_raw(raw).unwrap(); + + let extras = data + .extras() + .expect("extras should parse even without notes/groups"); + assert!(extras.notes.is_none()); + assert!(extras.groups.is_none()); + } + + #[test] + fn flow_data_extras_lost_after_flow_value_roundtrip() { + // Demonstrates the bug: serializing through FlowValue drops notes/groups. + // This is the root cause of #8641. + let raw = serde_json::value::to_raw_value(&json!({ + "modules": [], + "notes": [{"id": "n1", "text": "t", "color": "blue", "type": "free"}] + })) + .unwrap(); + + let data = FlowData::from_raw(raw).unwrap(); + + // Re-serialize through FlowValue (what RunFlowDependenciesRequest does) + let stripped = serde_json::to_string(data.value()).unwrap(); + let stripped_raw = RawValue::from_string(stripped).unwrap(); + let data2 = FlowData::from_raw(stripped_raw).unwrap(); + + // Notes are gone after the FlowValue round-trip + let extras = data2.extras().expect("extras should parse"); + assert!( + extras.notes.is_none(), + "notes lost after FlowValue round-trip" + ); + } +} diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index ab4bdee7b6..a0923ccabf 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -8,6 +8,8 @@ pub use windmill_types::flows::*; +use anyhow::Context; +use serde::Deserialize; use serde::Serialize; use sqlx::types::Json; use sqlx::types::JsonRawValue; @@ -15,10 +17,89 @@ use sqlx::types::JsonRawValue; use crate::{ cache::{self, FlowExtras}, db::DB, - error::Error, + error::{to_anyhow, Error}, + utils::{http_get_from_hub, StripPath}, worker::{to_raw_value, Connection}, + DEFAULT_HUB_BASE_URL, HUB_BASE_URL, PRIVATE_HUB_MIN_VERSION, }; +#[derive(Deserialize)] +pub struct HubFlow { + pub value: FlowValue, +} + +#[derive(Deserialize)] +struct HubFlowResponse { + flow: HubFlow, +} + +fn extract_hub_flow_id_from_path(path: &str) -> Result { + let hub_flow_path = path.strip_prefix("hub/flows/").ok_or_else(|| { + Error::BadRequest(format!( + "expected hub flow path to start with hub/flows/ (got {path})" + )) + })?; + + let flow_id = hub_flow_path + .split('/') + .next() + .filter(|segment| !segment.is_empty()) + .ok_or_else(|| { + Error::BadRequest(format!( + "expected hub flow path to include a numeric id after hub/flows/ (got {path})" + )) + })?; + + let flow_id = flow_id.parse::().map_err(|_| { + Error::BadRequest(format!( + "expected hub flow path to include a numeric id after hub/flows/ (got {path})" + )) + })?; + + if flow_id <= 0 { + return Err(Error::BadRequest(format!( + "expected hub flow path to include a positive numeric id after hub/flows/ (got {path})" + ))); + } + + Ok(flow_id) +} + +pub async fn get_full_hub_flow_by_path( + path: StripPath, + http_client: &reqwest::Client, + db: Option<&DB>, +) -> crate::error::Result { + let path = path.to_path(); + let flow_id = extract_hub_flow_id_from_path(&path)?; + let hub_base_url = HUB_BASE_URL.read().await.clone(); + let hub_url = format!("{hub_base_url}/flows/{flow_id}/json"); + + let response = match http_get_from_hub(http_client, &hub_url, false, None, db) + .await? + .error_for_status() + .map_err(to_anyhow) + { + Ok(response) => response, + Err(_) if hub_base_url != DEFAULT_HUB_BASE_URL && flow_id < PRIVATE_HUB_MIN_VERSION => + { + tracing::info!("Not found on private hub, fallback to default hub for hub flow {path}"); + let fallback_url = format!("{DEFAULT_HUB_BASE_URL}/flows/{flow_id}/json"); + http_get_from_hub(http_client, &fallback_url, false, None, db) + .await? + .error_for_status() + .map_err(to_anyhow)? + } + Err(err) => return Err(err.into()), + }; + + Ok(response + .json::() + .await + .context(format!("Decoding hub response for flow at path {path}"))? + .flow) +} + /// Serialize-only wrapper that combines resolved FlowValue with display-only extras. /// flatten + RawValue is fine for serialization (only deserialization breaks). #[derive(Serialize)] @@ -228,4 +309,36 @@ mod tests { assert!(!output.contains("notes")); assert!(!output.contains("groups")); } + + #[test] + fn extract_hub_flow_id_accepts_id_only_paths() { + assert_eq!(extract_hub_flow_id_from_path("hub/flows/76").unwrap(), 76); + } + + #[test] + fn extract_hub_flow_id_accepts_id_and_slug_paths() { + assert_eq!( + extract_hub_flow_id_from_path("hub/flows/76/send-message-to-company-ai-assistant") + .unwrap(), + 76 + ); + } + + #[test] + fn extract_hub_flow_id_rejects_non_numeric_ids() { + let err = extract_hub_flow_id_from_path("hub/flows/send_message").unwrap_err(); + assert!(matches!(err, Error::BadRequest(_))); + } + + #[test] + fn extract_hub_flow_id_rejects_missing_ids() { + let err = extract_hub_flow_id_from_path("hub/flows/").unwrap_err(); + assert!(matches!(err, Error::BadRequest(_))); + } + + #[test] + fn extract_hub_flow_id_rejects_zero_ids() { + let err = extract_hub_flow_id_from_path("hub/flows/0").unwrap_err(); + assert!(matches!(err, Error::BadRequest(_))); + } } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 31aadb8210..95e41c3bcd 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -1,6 +1,7 @@ pub const CUSTOM_TAGS_SETTING: &str = "custom_tags"; pub const DEFAULT_TAGS_PER_WORKSPACE_SETTING: &str = "default_tags_per_workspace"; pub const DEFAULT_TAGS_WORKSPACES_SETTING: &str = "default_tags_workspaces"; +pub const PREVIEW_TAGS_OVERRIDE_SETTING: &str = "preview_tags_override"; pub const BASE_URL_SETTING: &str = "base_url"; pub const WS_BASE_URL_SETTING: &str = "ws_base_url"; pub const OAUTH_SETTING: &str = "oauths"; @@ -64,6 +65,7 @@ pub const MIN_KEEP_ALIVE_VERSION_SETTING: &str = "min_keep_alive_version"; pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app"; pub const INSTANCE_EVENTS_WEBHOOK_SETTING: &str = "instance_events_webhook"; pub const WORKSPACE_REGISTRIES_SETTING: &str = "workspace_registries"; +pub const RESTART_COORDINATION_SETTING: &str = "_restart_coordination"; use std::sync::Arc; use tokio::sync::RwLock; @@ -98,6 +100,7 @@ pub const ENV_SETTINGS: &[&str] = &[ "BUNDLE_PATH", "GEM_PATH", "RUBY_CONCURRENT_DOWNLOADS", + "RSCRIPT_PATH", // for related places search: ADD_NEW_LANG "GOPRIVATE", "GOPROXY", diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index c843cc6621..ac1fe24a4e 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -243,6 +243,8 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub default_tags_per_workspace: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub preview_tags_override: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub disable_hub: Option, // String settings @@ -595,6 +597,7 @@ pub enum ScriptLang { Nu, Java, Ruby, + Rlang, } // --------------------------------------------------------------------------- @@ -870,6 +873,7 @@ pub const HIDDEN_SETTINGS: &[&str] = &[ "uid", "min_keep_alive_version", "automate_username_creation", + "_restart_coordination", ]; /// Top-level settings whose entire value is sensitive and must be fully redacted in logs. diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 583914c573..d5521288bd 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -14,6 +14,7 @@ use crate::{ client::AuthedClient, db::{AuthedRef, UserDbWithAuthed, DB}, error::{self, to_anyhow, Error}, + flows::get_full_hub_flow_by_path, get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, users::username_to_permissioned_as, @@ -154,15 +155,31 @@ pub async fn get_payload_tag_from_prefixed_path( .await? } else if path.starts_with("flow/") { let path = path.strip_prefix("flow/").unwrap().to_string(); - let FlowVersionInfo { dedicated_worker, tag, version, .. } = - get_latest_flow_version_info_for_path(None, &db, w_id, &path, true).await?; - ( - JobPayload::Flow { path, dedicated_worker, apply_preprocessor: false, version }, - tag, - None, - None, - None, - ) + if path.starts_with("hub/flows/") { + let hub_flow = + get_full_hub_flow_by_path(StripPath(path.clone()), &HTTP_CLIENT, Some(db)).await?; + ( + JobPayload::RawFlow { + value: hub_flow.value, + path: Some(path), + restarted_from: None, + }, + None, + None, + None, + None, + ) + } else { + let FlowVersionInfo { dedicated_worker, tag, version, .. } = + get_latest_flow_version_info_for_path(None, &db, w_id, &path, true).await?; + ( + JobPayload::Flow { path, dedicated_worker, apply_preprocessor: false, version }, + tag, + None, + None, + None, + ) + } } else { return Err(Error::BadRequest(format!( "path must start with script/ or flow/ (got {})", diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index b5ec518315..98fb7bdebf 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -69,6 +69,7 @@ pub mod git_sync_ee; pub mod git_sync_oss; pub mod jobs; pub mod jwt; +pub mod login_rate_limit; pub mod more_serde; pub mod oauth2; #[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] @@ -406,6 +407,8 @@ pub struct PgDatabase { pub sslmode: Option, pub dbname: String, pub root_certificate_pem: Option, + pub use_iam_auth: Option, + pub region: Option, } // Wrapper enum to hold either Tls or NoTls connection @@ -513,6 +516,75 @@ impl PgDatabase { } } + #[cfg(all(feature = "enterprise", feature = "private"))] + pub async fn connect_with_iam( + &self, + ) -> Result<(tokio_postgres::Client, TokioPgConnection), error::Error> { + use native_tls::TlsConnector; + use postgres_native_tls::MakeTlsConnector; + + // Resolve region: resource field takes priority, then env var + let region = match self.region.as_deref() { + Some(r) => r.to_string(), + None => std::env::var("AWS_REGION").map_err(|_| { + error::Error::BadConfig( + "Region is required for IAM RDS auth. Set 'region' on the resource or AWS_REGION env var".to_string(), + ) + })?, + }; + + let port = self.port.unwrap_or(5432); + let user = self.user.as_deref().unwrap_or("postgres"); + + let token = db_iam_ee::generate_auth_token(®ion, &self.host, port as u64, user) + .await + .map_err(|e| { + error::Error::InternalErr(format!("IAM token generation failed: {e:#}")) + })?; + + // RDS IAM auth requires SSL + let mut connector = TlsConnector::builder(); + if let Some(root_certificate_pem) = &self.root_certificate_pem { + if !root_certificate_pem.is_empty() { + connector.add_root_certificate( + native_tls::Certificate::from_pem(root_certificate_pem.as_bytes()) + .map_err(|e| error::Error::BadConfig(format!("Invalid Certs: {e:#}")))?, + ); + } else { + connector.danger_accept_invalid_certs(true); + connector.danger_accept_invalid_hostnames(true); + } + } else { + tracing::warn!("IAM RDS auth without root certificate: TLS certificate verification is disabled. Consider providing root_certificate_pem for production use."); + connector + .danger_accept_invalid_certs(true) + .danger_accept_invalid_hostnames(true); + } + + tracing::info!("Creating new IAM RDS connection to {}", &self.host); + + // Use Config builder directly to pass the IAM token as the password. + // This avoids needing to URL-encode the token into a connection string. + let mut config = tokio_postgres::Config::new(); + config + .host(&self.host) + .port(port as u16) + .user(user) + .password(&token) + .dbname(&self.dbname) + .ssl_mode(tokio_postgres::config::SslMode::Require); + + let (client, connection) = tokio::time::timeout( + std::time::Duration::from_secs(20), + config.connect(MakeTlsConnector::new(connector.build().map_err(to_anyhow)?)), + ) + .await + .map_err(to_anyhow)? + .map_err(to_anyhow)?; + + Ok((client, TokioPgConnection::Tls(connection))) + } + pub fn parse_uri(url: &str) -> Result { let parsed_url = url::Url::parse(url) .map_err(|_| Error::BadConfig("Invalid PostgreSQL URL".to_string()))?; @@ -551,6 +623,8 @@ impl PgDatabase { dbname, sslmode, root_certificate_pem: None, + use_iam_auth: None, + region: None, }) } } diff --git a/backend/windmill-common/src/login_rate_limit.rs b/backend/windmill-common/src/login_rate_limit.rs new file mode 100644 index 0000000000..aede8df2e4 --- /dev/null +++ b/backend/windmill-common/src/login_rate_limit.rs @@ -0,0 +1,206 @@ +use chrono::Utc; +use dashmap::DashMap; +use hyper::StatusCode; +use std::sync::atomic::{AtomicI32, AtomicI64, AtomicU64, Ordering}; +use std::sync::LazyLock; + +use crate::error::{Error, Result}; +use crate::worker::CLOUD_HOSTED; + +const DEFAULT_PER_IP_LIMIT: i32 = 120; +const DEFAULT_PER_ACCOUNT_LIMIT: i32 = 30; +const DEFAULT_GLOBAL_LIMIT: i32 = 10000; +const EVICTION_INTERVAL: u64 = 256; + +struct RateLimitEntry { + count: i32, + minute_bucket: i64, +} + +static IP_RATE_LIMIT: LazyLock> = LazyLock::new(DashMap::new); +static ACCOUNT_RATE_LIMIT: LazyLock> = LazyLock::new(DashMap::new); + +static GLOBAL_COUNT: AtomicI32 = AtomicI32::new(0); +static GLOBAL_MINUTE: AtomicI64 = AtomicI64::new(0); + +static EVICTION_COUNTER: AtomicU64 = AtomicU64::new(0); + +static PER_IP_LIMIT: LazyLock = LazyLock::new(|| { + std::env::var("LOGIN_RATE_LIMIT_PER_IP") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_PER_IP_LIMIT) +}); + +static PER_IP_LIMIT_EXPLICIT: LazyLock = LazyLock::new(|| { + std::env::var("LOGIN_RATE_LIMIT_PER_IP") + .ok() + .and_then(|v| v.parse::().ok()) + .is_some() +}); + +static PER_ACCOUNT_LIMIT: LazyLock = LazyLock::new(|| { + std::env::var("LOGIN_RATE_LIMIT_PER_ACCOUNT") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_PER_ACCOUNT_LIMIT) +}); + +static PER_ACCOUNT_LIMIT_EXPLICIT: LazyLock = LazyLock::new(|| { + std::env::var("LOGIN_RATE_LIMIT_PER_ACCOUNT") + .ok() + .and_then(|v| v.parse::().ok()) + .is_some() +}); + +static GLOBAL_LIMIT: LazyLock = LazyLock::new(|| { + std::env::var("LOGIN_RATE_LIMIT_GLOBAL") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_GLOBAL_LIMIT) +}); + +/// Extract client IP from proxy headers. Only meaningful when behind a trusted +/// reverse proxy (e.g. CLOUD_HOSTED). Returns `None` if no proxy header is present. +pub fn extract_client_ip(headers: &axum::http::HeaderMap) -> Option { + if let Some(real_ip) = headers.get("x-real-ip") { + if let Ok(ip) = real_ip.to_str() { + let trimmed = ip.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + + if let Some(forwarded_for) = headers.get("x-forwarded-for") { + if let Ok(ips) = forwarded_for.to_str() { + if let Some(first_ip) = ips.split(',').next() { + let trimmed = first_ip.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + } + + None +} + +fn maybe_evict(maps: &[&DashMap], current_minute: i64) { + let count = EVICTION_COUNTER.fetch_add(1, Ordering::Relaxed); + if count % EVICTION_INTERVAL == 0 { + for map in maps { + map.retain(|_, v| v.minute_bucket >= current_minute - 1); + } + } +} + +/// Atomically check the rate limit and increment the counter. Follows the +/// `public_app_rate_limit.rs` pattern — the DashMap entry lock is held across +/// both the check and the increment, preventing TOCTOU races. +fn check_and_increment( + map: &DashMap, + key: &str, + limit: i32, + current_minute: i64, +) -> Result<()> { + let mut entry = map + .entry(key.to_string()) + .or_insert(RateLimitEntry { count: 0, minute_bucket: current_minute }); + + if entry.minute_bucket != current_minute { + entry.count = 0; + entry.minute_bucket = current_minute; + } + + if entry.count >= limit { + return Err(Error::Generic( + StatusCode::TOO_MANY_REQUESTS, + "Too many login attempts. Please try again later.".to_string(), + )); + } + + entry.count += 1; + Ok(()) +} + +fn record_failure(map: &DashMap, key: &str) { + let current_minute = Utc::now().timestamp() / 60; + + let mut entry = map + .entry(key.to_string()) + .or_insert(RateLimitEntry { count: 0, minute_bucket: current_minute }); + + if entry.minute_bucket != current_minute { + entry.count = 1; + entry.minute_bucket = current_minute; + } else { + entry.count += 1; + } +} + +/// Called BEFORE authentication. Checks and increments global + per-IP counters. +/// The global counter counts all login attempts (not just failures), so it acts as +/// a general throttle on login traffic per server instance. +/// Per-IP is only active on CLOUD_HOSTED or when LOGIN_RATE_LIMIT_PER_IP is explicitly set. +pub fn check_and_increment_login_attempt( + headers: &axum::http::HeaderMap, + email: &str, +) -> Result<()> { + let current_minute = Utc::now().timestamp() / 60; + maybe_evict(&[&IP_RATE_LIMIT, &ACCOUNT_RATE_LIMIT], current_minute); + + // Global limit: always on, uses atomics (single key, no need for DashMap) + check_and_increment_global(current_minute)?; + + // Per-IP limit: CLOUD_HOSTED or explicit opt-in + if *CLOUD_HOSTED || *PER_IP_LIMIT_EXPLICIT { + if let Some(ip) = extract_client_ip(headers) { + check_and_increment(&IP_RATE_LIMIT, &ip, *PER_IP_LIMIT, current_minute)?; + } + } + + // Per-account check (read-only, does not increment — failures are recorded separately) + if *CLOUD_HOSTED || *PER_ACCOUNT_LIMIT_EXPLICIT { + let entry = ACCOUNT_RATE_LIMIT.get(email); + if let Some(entry) = entry { + if entry.minute_bucket == current_minute && entry.count >= *PER_ACCOUNT_LIMIT { + return Err(Error::Generic( + StatusCode::TOO_MANY_REQUESTS, + "Too many login attempts. Please try again later.".to_string(), + )); + } + } + } + + Ok(()) +} + +fn check_and_increment_global(current_minute: i64) -> Result<()> { + let stored_minute = GLOBAL_MINUTE.load(Ordering::Relaxed); + if stored_minute != current_minute { + // Minute rolled over — reset. Race here is benign: worst case two threads + // both reset, and we lose a few counts at the boundary. + GLOBAL_MINUTE.store(current_minute, Ordering::Relaxed); + GLOBAL_COUNT.store(1, Ordering::Relaxed); + return Ok(()); + } + + let count = GLOBAL_COUNT.fetch_add(1, Ordering::Relaxed); + if count >= *GLOBAL_LIMIT { + return Err(Error::Generic( + StatusCode::TOO_MANY_REQUESTS, + "Too many login attempts. Please try again later.".to_string(), + )); + } + + Ok(()) +} + +/// Called AFTER authentication failure. Records per-account failure. +/// Per-account is only active on CLOUD_HOSTED or when LOGIN_RATE_LIMIT_PER_ACCOUNT is explicitly set. +pub fn record_login_failure(email: &str) { + if *CLOUD_HOSTED || *PER_ACCOUNT_LIMIT_EXPLICIT { + record_failure(&ACCOUNT_RATE_LIMIT, email); + } +} diff --git a/backend/windmill-common/src/otel_oss.rs b/backend/windmill-common/src/otel_oss.rs index 3710464607..27c7101dc0 100644 --- a/backend/windmill-common/src/otel_oss.rs +++ b/backend/windmill-common/src/otel_oss.rs @@ -59,7 +59,7 @@ pub(crate) fn init_otlp_tracer( _mode: &Mode, _hostname: &str, _env: &str, -) -> Option { +) -> Option { None } diff --git a/backend/windmill-common/src/query_builders.rs b/backend/windmill-common/src/query_builders.rs index c52707823e..6d2c9519a5 100644 --- a/backend/windmill-common/src/query_builders.rs +++ b/backend/windmill-common/src/query_builders.rs @@ -39,7 +39,14 @@ fn deserialize_string_from_null<'de, D>(deserializer: D) -> Result, { - Option::::deserialize(deserializer).map(|v| v.unwrap_or_default()) + // DuckDB may return booleans for fields that other databases return as strings + let v = serde_json::Value::deserialize(deserializer)?; + match v { + serde_json::Value::Null => Ok(String::new()), + serde_json::Value::String(s) => Ok(s), + serde_json::Value::Bool(b) => Ok(b.to_string()), + other => Ok(other.to_string()), + } } fn deserialize_column_identity_from_null<'de, D>( @@ -49,15 +56,21 @@ where D: Deserializer<'de>, { // MySQL returns uppercase "YES"/"NO" while the enum expects title case. - let v = Option::::deserialize(deserializer)?; - match v.as_deref() { - None => Ok(ColumnIdentity::default()), - Some(s) => match s.to_lowercase().as_str() { + // DuckDB returns a boolean false instead of a string. + let v = serde_json::Value::deserialize(deserializer)?; + match v { + serde_json::Value::Null => Ok(ColumnIdentity::default()), + serde_json::Value::Bool(_) => Ok(ColumnIdentity::No), + serde_json::Value::String(s) => match s.to_lowercase().as_str() { "no" => Ok(ColumnIdentity::No), "yes" | "always" => Ok(ColumnIdentity::Always), "by default" => Ok(ColumnIdentity::ByDefault), _ => Ok(ColumnIdentity::No), }, + _ => Err(serde::de::Error::custom(format!( + "expected string, bool, or null for isidentity, got {}", + v + ))), } } @@ -2369,7 +2382,7 @@ fn make_load_table_metadata_query( COLUMN_DEFAULT as DefaultValue, false as IsPrimaryKey, false as IsIdentity, - IS_NULLABLE as IsNullable, + CASE WHEN IS_NULLABLE = true THEN 'YES' ELSE 'NO' END as IsNullable, false as IsEnum, TABLE_NAME as table_name FROM information_schema.columns c diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index d3035b1b49..50790fe65b 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -428,6 +428,7 @@ pub async fn clone_script<'c>( preserve_on_behalf_of: None, assets: s.assets, modules: s.modules, + auto_parent: None, }; let new_hash = hash_script(&ns); diff --git a/backend/windmill-common/src/sensitive_log_masks.rs b/backend/windmill-common/src/sensitive_log_masks.rs index 8123d6d55b..b6f6262b77 100644 --- a/backend/windmill-common/src/sensitive_log_masks.rs +++ b/backend/windmill-common/src/sensitive_log_masks.rs @@ -88,8 +88,16 @@ pub fn snapshot(job_id: &Uuid) -> Option { let replacements: Vec = sorted .iter() .map(|s| { - let prefix: String = s.chars().take(3).collect(); - format!("{}*****", prefix) + let char_count = s.chars().count(); + if char_count > 20 { + let prefix: String = s.chars().take(3).collect(); + let suffix: String = s.chars().skip(char_count - 3).collect(); + format!("{}*****{}", prefix, suffix) + } else { + let first: String = s.chars().take(1).collect(); + let last: String = s.chars().skip(char_count - 1).collect(); + format!("{}*****{}", first, last) + } }) .collect(); diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 3b009a36eb..1a2ca33518 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -273,3 +273,4 @@ where } } } + diff --git a/backend/windmill-common/src/users.rs b/backend/windmill-common/src/users.rs index 453466ac7d..f91c21701a 100644 --- a/backend/windmill-common/src/users.rs +++ b/backend/windmill-common/src/users.rs @@ -81,6 +81,46 @@ pub async fn get_email_from_permissioned_as( } } +/// Compute the highest-precedence workspace role for a user across all their instance groups. +/// +/// Precedence: admin (3) > developer (2) > operator (1). +/// Returns `(best_group_name, is_admin, is_operator)`. +pub fn compute_highest_workspace_role( + user_igroups: &[String], + ws_configured_groups: &[String], + ws_roles: &std::collections::HashMap, +) -> (String, bool, bool) { + let mut best_group = String::new(); + let mut best_precedence = 0u8; + + for group in user_igroups { + if !ws_configured_groups.contains(group) { + continue; + } + let default_role = "developer".to_string(); + let role = ws_roles.get(group).unwrap_or(&default_role); + let precedence = match role.as_str() { + "admin" => 3u8, + "operator" => 1, + _ => 2, + }; + if precedence > best_precedence { + best_precedence = precedence; + best_group = group.clone(); + } + } + + let default_role = "developer".to_string(); + let best_role_str = ws_roles.get(&best_group).unwrap_or(&default_role); + let (is_admin, is_operator) = match best_role_str.as_str() { + "admin" => (true, false), + "operator" => (false, true), + _ => (false, false), + }; + + (best_group, is_admin, is_operator) +} + pub fn truncate_token(token: &str) -> String { if token.len() > 10 { let mut s = token[..10].to_owned(); @@ -105,4 +145,63 @@ mod tests { assert_eq!(username_to_permissioned_as("group-all"), "g/all"); assert_eq!(username_to_permissioned_as("group-my-team"), "g/my-team"); } + + #[test] + fn test_compute_highest_workspace_role_admin_wins() { + let user_groups = vec!["ops".to_string(), "admins".to_string()]; + let ws_groups = vec!["ops".to_string(), "admins".to_string()]; + let mut roles = std::collections::HashMap::new(); + roles.insert("ops".to_string(), "operator".to_string()); + roles.insert("admins".to_string(), "admin".to_string()); + + let (group, is_admin, is_operator) = + compute_highest_workspace_role(&user_groups, &ws_groups, &roles); + assert_eq!(group, "admins"); + assert!(is_admin); + assert!(!is_operator); + } + + #[test] + fn test_compute_highest_workspace_role_developer_over_operator() { + let user_groups = vec!["devs".to_string(), "ops".to_string()]; + let ws_groups = vec!["devs".to_string(), "ops".to_string()]; + let mut roles = std::collections::HashMap::new(); + roles.insert("devs".to_string(), "developer".to_string()); + roles.insert("ops".to_string(), "operator".to_string()); + + let (group, is_admin, is_operator) = + compute_highest_workspace_role(&user_groups, &ws_groups, &roles); + assert_eq!(group, "devs"); + assert!(!is_admin); + assert!(!is_operator); + } + + #[test] + fn test_compute_highest_workspace_role_skips_unconfigured_groups() { + let user_groups = vec!["admins".to_string(), "other".to_string()]; + let ws_groups = vec!["ops".to_string()]; // admins not configured for this workspace + let mut roles = std::collections::HashMap::new(); + roles.insert("admins".to_string(), "admin".to_string()); + roles.insert("ops".to_string(), "operator".to_string()); + + let (group, is_admin, is_operator) = + compute_highest_workspace_role(&user_groups, &ws_groups, &roles); + // No user groups match ws_configured_groups, so best_group stays empty + assert_eq!(group, ""); + assert!(!is_admin); + assert!(!is_operator); + } + + #[test] + fn test_compute_highest_workspace_role_defaults_to_developer() { + let user_groups = vec!["team".to_string()]; + let ws_groups = vec!["team".to_string()]; + let roles = std::collections::HashMap::new(); // no role configured → developer + + let (group, is_admin, is_operator) = + compute_highest_workspace_role(&user_groups, &ws_groups, &roles); + assert_eq!(group, "team"); + assert!(!is_admin); + assert!(!is_operator); + } } diff --git a/backend/windmill-common/src/webhook.rs b/backend/windmill-common/src/webhook.rs index caf8ec08fc..d695ba1f5a 100644 --- a/backend/windmill-common/src/webhook.rs +++ b/backend/windmill-common/src/webhook.rs @@ -233,6 +233,9 @@ impl WebhookShared { } pub fn send_message(&self, workspace_id: String, message: WebhookMessage) { + if *crate::worker::CLOUD_HOSTED { + return; + } let _ = self.channel.send(WebhookPayload::WorkspaceEvent( workspace_id.clone(), message, diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index b1ba371ea2..7a200fb44b 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -184,6 +184,7 @@ lazy_static::lazy_static! { "nu".to_string(), "java".to_string(), "ruby".to_string(), + "rlang".to_string(), "duckdb".to_string(), // for related places search: ADD_NEW_LANG "dependency".to_string(), @@ -205,6 +206,7 @@ lazy_static::lazy_static! { pub static ref DEFAULT_TAGS_PER_WORKSPACE: AtomicBool = AtomicBool::new(false); pub static ref DEFAULT_TAGS_WORKSPACES: Arc>>> = Arc::new(RwLock::new(None)); + pub static ref PREVIEW_TAGS_OVERRIDE: AtomicBool = AtomicBool::new(false); pub static ref MAX_TIMEOUT: u64 = std::env::var("TIMEOUT") .ok() @@ -727,6 +729,13 @@ pub struct RubyAnnotations { pub verbose: bool, } +#[annotations("#")] +pub struct RlangAnnotations { + pub renv_verbose: bool, + pub renv_install_verbose: bool, + pub sandbox: bool, +} + #[annotations("#")] pub struct PythonAnnotations { pub no_cache: bool, diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 444df60a0c..87d2b8917d 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -149,7 +149,7 @@ pub enum ObjectType { WorkspaceDependencies, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28180/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28186/sync-script-to-git-repo-windmill"; #[derive(Serialize, Deserialize, Debug)] pub struct GitRepositorySettings { diff --git a/backend/windmill-common/tests/instance_group_auto_add.rs b/backend/windmill-common/tests/instance_group_auto_add.rs index e8a6c841c6..95b20cc842 100644 --- a/backend/windmill-common/tests/instance_group_auto_add.rs +++ b/backend/windmill-common/tests/instance_group_auto_add.rs @@ -28,10 +28,14 @@ mod tests { use serde_json::json; use sqlx::{Pool, Postgres}; + use windmill_common::users::compute_highest_workspace_role; /// Test that configuring instance groups for a workspace auto-adds existing group members #[ignore = "requires database setup - run with --ignored flag"] - #[sqlx::test(migrations = "../migrations", fixtures("base", "instance_group_auto_add"))] + #[sqlx::test( + migrations = "../migrations", + fixtures("base", "instance_group_auto_add") + )] async fn test_configure_instance_groups_adds_existing_members(db: Pool) { // Configure workspace to auto-add users from 'engineering' group with 'developer' role let groups = vec!["engineering".to_string()]; @@ -112,8 +116,14 @@ mod tests { "Alice should be in the workspace" ); let alice = alice_in_workspace.unwrap(); - assert!(!alice.is_admin, "Alice should not be admin (developer role)"); - assert!(!alice.operator, "Alice should not be operator (developer role)"); + assert!( + !alice.is_admin, + "Alice should not be admin (developer role)" + ); + assert!( + !alice.operator, + "Alice should not be operator (developer role)" + ); // Check added_via field let added_via = alice.added_via.expect("added_via should be set"); @@ -158,7 +168,10 @@ mod tests { /// Test role assignment based on instance group configuration #[ignore = "requires database setup - run with --ignored flag"] - #[sqlx::test(migrations = "../migrations", fixtures("base", "instance_group_auto_add"))] + #[sqlx::test( + migrations = "../migrations", + fixtures("base", "instance_group_auto_add") + )] async fn test_role_assignment_admin(db: Pool) { // Configure workspace with admins group having admin role let groups = vec!["admins".to_string()]; @@ -209,7 +222,10 @@ mod tests { /// Test role assignment for operator #[ignore = "requires database setup - run with --ignored flag"] - #[sqlx::test(migrations = "../migrations", fixtures("base", "instance_group_auto_add"))] + #[sqlx::test( + migrations = "../migrations", + fixtures("base", "instance_group_auto_add") + )] async fn test_role_assignment_operator(db: Pool) { // Configure workspace with sales group having operator role let groups = vec!["sales".to_string()]; @@ -260,7 +276,10 @@ mod tests { /// Test role precedence when user is in multiple instance groups #[ignore = "requires database setup - run with --ignored flag"] - #[sqlx::test(migrations = "../migrations", fixtures("base", "instance_group_auto_add"))] + #[sqlx::test( + migrations = "../migrations", + fixtures("base", "instance_group_auto_add") + )] async fn test_role_precedence_multiple_groups(db: Pool) { // Configure workspace with multiple groups: engineering (admin), sales (operator) // Bob is in both groups, should get admin role (highest precedence) @@ -306,7 +325,10 @@ mod tests { .await .expect("Failed to query user"); - assert!(bob.is_admin, "Bob should be admin (highest precedence role)"); + assert!( + bob.is_admin, + "Bob should be admin (highest precedence role)" + ); assert!(!bob.operator, "Bob should not be operator"); // Verify added_via tracks the primary group (engineering, the one with highest precedence) @@ -320,9 +342,305 @@ mod tests { println!("✓ Role precedence works correctly for users in multiple groups"); } + /// Test that adding a user to a second instance group upgrades their workspace role + /// if the new group has a higher-precedence role. + /// This is a regression test for the bug where only the newly-added group's role was used. + #[ignore = "requires database setup - run with --ignored flag"] + #[sqlx::test( + migrations = "../migrations", + fixtures("base", "instance_group_auto_add") + )] + async fn test_role_upgrade_when_added_to_higher_group(db: Pool) { + // Configure workspace: engineering=operator, admins=admin + let groups = vec!["engineering".to_string(), "admins".to_string()]; + let roles = json!({"engineering": "operator", "admins": "admin"}); + + sqlx::query!( + r#" + UPDATE workspace_settings + SET auto_invite = jsonb_build_object( + 'instance_groups', $2::jsonb, + 'instance_groups_roles', $3::jsonb + ) + WHERE workspace_id = $1 + "#, + "ws-multi-group", + serde_json::to_value(&groups).unwrap(), + &roles, + ) + .execute(&db) + .await + .expect("Failed to update workspace settings"); + + // Step 1: Alice is added via engineering group (operator) + // (alice is already in engineering from fixture) + let added_via = json!({"source": "instance_group", "group": "engineering"}); + sqlx::query!( + "INSERT INTO usr (workspace_id, username, email, is_admin, operator, added_via) + VALUES ($1, 'alice', 'alice@example.com', false, true, $2)", + "ws-multi-group", + &added_via, + ) + .execute(&db) + .await + .expect("Failed to add user"); + + sqlx::query!( + "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, 'alice', 'all')", + "ws-multi-group", + ) + .execute(&db) + .await + .expect("Failed to add user to all group"); + + // Verify initial state: alice is operator + let alice = sqlx::query!( + "SELECT is_admin, operator FROM usr WHERE workspace_id = 'ws-multi-group' AND email = 'alice@example.com'" + ) + .fetch_one(&db) + .await + .expect("Failed to query user"); + assert!(!alice.is_admin, "Alice should start as non-admin"); + assert!(alice.operator, "Alice should start as operator"); + + // Step 2: Alice is added to admins group + sqlx::query!( + "INSERT INTO email_to_igroup (email, igroup) VALUES ('alice@example.com', 'admins') ON CONFLICT DO NOTHING" + ) + .execute(&db) + .await + .expect("Failed to add to admins group"); + + // Step 3: Simulate the fixed logic — find all user's groups, compute highest role, update + let user_igroups: Vec = sqlx::query_scalar!( + "SELECT igroup FROM email_to_igroup WHERE email = 'alice@example.com'" + ) + .fetch_all(&db) + .await + .expect("Failed to fetch user groups"); + + let ws = sqlx::query!( + r#" + SELECT auto_invite->'instance_groups_roles' as instance_groups_roles, + auto_invite->'instance_groups' as instance_groups_json + FROM workspace_settings WHERE workspace_id = 'ws-multi-group' + "#, + ) + .fetch_one(&db) + .await + .expect("Failed to fetch workspace settings"); + + let ws_roles: std::collections::HashMap = ws + .instance_groups_roles + .and_then(|r| serde_json::from_value(r).ok()) + .unwrap_or_default(); + + let ws_configured_groups: Vec = ws + .instance_groups_json + .and_then(|ig| serde_json::from_value(ig).ok()) + .unwrap_or_default(); + + let (best_group, is_admin, is_operator) = + compute_highest_workspace_role(&user_igroups, &ws_configured_groups, &ws_roles); + + let instance_group_source = json!({ + "source": "instance_group", + "group": &best_group + }); + + sqlx::query!( + "UPDATE usr SET is_admin = $1, operator = $2, added_via = $3 WHERE workspace_id = $4 AND email = $5 AND added_via->>'source' = 'instance_group'", + is_admin, + is_operator, + &instance_group_source, + "ws-multi-group", + "alice@example.com" + ) + .execute(&db) + .await + .expect("Failed to update user role"); + + // Verify: alice should now be admin (highest precedence) + let alice = sqlx::query!( + "SELECT is_admin, operator, added_via FROM usr WHERE workspace_id = 'ws-multi-group' AND email = 'alice@example.com'" + ) + .fetch_one(&db) + .await + .expect("Failed to query user"); + + assert!(alice.is_admin, "Alice should be upgraded to admin"); + assert!(!alice.operator, "Alice should no longer be operator"); + + let added_via = alice.added_via.expect("added_via should be set"); + assert_eq!( + added_via.get("group").and_then(|v| v.as_str()), + Some("admins"), + "added_via should track the admin group (highest precedence)" + ); + + println!("✓ Role is upgraded when user is added to a higher-precedence group"); + } + + /// Test that adding a user to a lower-precedence group does NOT downgrade their role + #[ignore = "requires database setup - run with --ignored flag"] + #[sqlx::test( + migrations = "../migrations", + fixtures("base", "instance_group_auto_add") + )] + async fn test_no_role_downgrade_when_added_to_lower_group(db: Pool) { + // Configure workspace: engineering=admin, sales=operator + let groups = vec!["engineering".to_string(), "sales".to_string()]; + let roles = json!({"engineering": "admin", "sales": "operator"}); + + sqlx::query!( + r#" + UPDATE workspace_settings + SET auto_invite = jsonb_build_object( + 'instance_groups', $2::jsonb, + 'instance_groups_roles', $3::jsonb + ) + WHERE workspace_id = $1 + "#, + "ws-multi-group", + serde_json::to_value(&groups).unwrap(), + &roles, + ) + .execute(&db) + .await + .expect("Failed to update workspace settings"); + + // Alice starts as admin from engineering + let added_via = json!({"source": "instance_group", "group": "engineering"}); + sqlx::query!( + "INSERT INTO usr (workspace_id, username, email, is_admin, operator, added_via) + VALUES ($1, 'alice', 'alice@example.com', true, false, $2)", + "ws-multi-group", + &added_via, + ) + .execute(&db) + .await + .expect("Failed to add user"); + + // Now simulate adding alice to sales group (operator — lower precedence) + // The fixed code should keep her as admin + let user_igroups = vec!["engineering".to_string(), "sales".to_string()]; + let ws_configured_groups = vec!["engineering".to_string(), "sales".to_string()]; + let ws_roles: std::collections::HashMap = + serde_json::from_value(roles).unwrap(); + + let (best_group, is_admin, is_operator) = + compute_highest_workspace_role(&user_igroups, &ws_configured_groups, &ws_roles); + + let instance_group_source = json!({"source": "instance_group", "group": &best_group}); + sqlx::query!( + "UPDATE usr SET is_admin = $1, operator = $2, added_via = $3 WHERE workspace_id = $4 AND email = $5 AND added_via->>'source' = 'instance_group'", + is_admin, is_operator, &instance_group_source, "ws-multi-group", "alice@example.com" + ) + .execute(&db) + .await + .expect("Failed to update user role"); + + let alice = sqlx::query!( + "SELECT is_admin, operator, added_via FROM usr WHERE workspace_id = 'ws-multi-group' AND email = 'alice@example.com'" + ) + .fetch_one(&db) + .await + .expect("Failed to query user"); + + assert!(alice.is_admin, "Alice should remain admin (not downgraded)"); + assert!(!alice.operator, "Alice should not become operator"); + assert_eq!( + alice + .added_via + .unwrap() + .get("group") + .and_then(|v| v.as_str()), + Some("engineering"), + "added_via should still track engineering (highest precedence)" + ); + + println!("✓ Role is NOT downgraded when user is added to a lower-precedence group"); + } + + /// Test that manually-added users are not affected by instance group role updates + #[ignore = "requires database setup - run with --ignored flag"] + #[sqlx::test( + migrations = "../migrations", + fixtures("base", "instance_group_auto_add") + )] + async fn test_manual_users_not_affected_by_group_role_update(db: Pool) { + // Configure workspace + let groups = vec!["engineering".to_string()]; + let roles = json!({"engineering": "operator"}); + + sqlx::query!( + r#" + UPDATE workspace_settings + SET auto_invite = jsonb_build_object( + 'instance_groups', $2::jsonb, + 'instance_groups_roles', $3::jsonb + ) + WHERE workspace_id = $1 + "#, + "ws-multi-group", + serde_json::to_value(&groups).unwrap(), + &roles, + ) + .execute(&db) + .await + .expect("Failed to update workspace settings"); + + // Alice was manually added as admin (no added_via) + sqlx::query!( + "INSERT INTO usr (workspace_id, username, email, is_admin, operator) + VALUES ('ws-multi-group', 'alice', 'alice@example.com', true, false)", + ) + .execute(&db) + .await + .expect("Failed to add user"); + + // The UPDATE with added_via->>'source' = 'instance_group' filter should NOT match + let instance_group_source = json!({"source": "instance_group", "group": "engineering"}); + let result = sqlx::query!( + "UPDATE usr SET is_admin = $1, operator = $2, added_via = $3 WHERE workspace_id = $4 AND email = $5 AND added_via->>'source' = 'instance_group'", + false, true, &instance_group_source, "ws-multi-group", "alice@example.com" + ) + .execute(&db) + .await + .expect("Failed to execute update"); + + assert_eq!( + result.rows_affected(), + 0, + "UPDATE should not affect manually-added users" + ); + + let alice = sqlx::query!( + "SELECT is_admin, operator, added_via FROM usr WHERE workspace_id = 'ws-multi-group' AND email = 'alice@example.com'" + ) + .fetch_one(&db) + .await + .expect("Failed to query user"); + + assert!(alice.is_admin, "Manually-added admin should remain admin"); + assert!( + !alice.operator, + "Manually-added admin should not become operator" + ); + assert!( + alice.added_via.is_none(), + "added_via should remain NULL for manual users" + ); + + println!("✓ Manually-added users are not affected by instance group role updates"); + } + /// Test removing user from instance group removes them from workspace #[ignore = "requires database setup - run with --ignored flag"] - #[sqlx::test(migrations = "../migrations", fixtures("base", "instance_group_auto_add"))] + #[sqlx::test( + migrations = "../migrations", + fixtures("base", "instance_group_auto_add") + )] async fn test_remove_user_from_instance_group(db: Pool) { // First, add alice to the workspace via engineering group let added_via = json!({"source": "instance_group", "group": "engineering"}); @@ -416,7 +734,10 @@ mod tests { /// Test that users added via domain are not affected by instance group removal #[ignore = "requires database setup - run with --ignored flag"] - #[sqlx::test(migrations = "../migrations", fixtures("base", "instance_group_auto_add"))] + #[sqlx::test( + migrations = "../migrations", + fixtures("base", "instance_group_auto_add") + )] async fn test_domain_added_users_not_affected_by_group_removal(db: Pool) { // Add alice via domain (not instance group) let added_via = json!({"source": "domain", "domain": "example.com"}); @@ -469,7 +790,10 @@ mod tests { /// Test cleanup when instance group is removed from workspace configuration #[ignore = "requires database setup - run with --ignored flag"] - #[sqlx::test(migrations = "../migrations", fixtures("base", "instance_group_auto_add"))] + #[sqlx::test( + migrations = "../migrations", + fixtures("base", "instance_group_auto_add") + )] async fn test_cleanup_removed_instance_groups(db: Pool) { // First, add users via engineering group for (username, email) in &[("alice", "alice@example.com"), ("bob", "bob@example.com")] { @@ -513,12 +837,11 @@ mod tests { // This should trigger cleanup of users added via that group // Get all users in the engineering group - let group_users = sqlx::query_scalar!( - "SELECT email FROM email_to_igroup WHERE igroup = 'engineering'" - ) - .fetch_all(&db) - .await - .expect("Failed to get group users"); + let group_users = + sqlx::query_scalar!("SELECT email FROM email_to_igroup WHERE igroup = 'engineering'") + .fetch_all(&db) + .await + .expect("Failed to get group users"); // Remove users who were added via engineering group for email in group_users { @@ -588,7 +911,10 @@ mod tests { /// Test that users are not duplicated if already in workspace #[ignore = "requires database setup - run with --ignored flag"] - #[sqlx::test(migrations = "../migrations", fixtures("base", "instance_group_auto_add"))] + #[sqlx::test( + migrations = "../migrations", + fixtures("base", "instance_group_auto_add") + )] async fn test_no_duplicate_users(db: Pool) { // Add alice to workspace first (without instance group tracking) sqlx::query!( @@ -636,7 +962,10 @@ mod tests { /// Test workspace without auto-add configured is not affected #[ignore = "requires database setup - run with --ignored flag"] - #[sqlx::test(migrations = "../migrations", fixtures("base", "instance_group_auto_add"))] + #[sqlx::test( + migrations = "../migrations", + fixtures("base", "instance_group_auto_add") + )] async fn test_workspace_without_auto_add_not_affected(db: Pool) { // ws-no-auto-add has no instance_groups configured @@ -672,7 +1001,10 @@ mod tests { /// Test querying workspaces configured with a specific instance group #[ignore = "requires database setup - run with --ignored flag"] - #[sqlx::test(migrations = "../migrations", fixtures("base", "instance_group_auto_add"))] + #[sqlx::test( + migrations = "../migrations", + fixtures("base", "instance_group_auto_add") + )] async fn test_query_workspaces_with_instance_group(db: Pool) { // Configure ws-with-auto-add to use engineering group let groups = vec!["engineering".to_string()]; @@ -708,7 +1040,11 @@ mod tests { .await .expect("Failed to query workspaces"); - assert_eq!(workspaces.len(), 1, "Should find 1 workspace with engineering group"); + assert_eq!( + workspaces.len(), + 1, + "Should find 1 workspace with engineering group" + ); assert_eq!(workspaces[0].workspace_id, "ws-with-auto-add"); // Verify the role configuration is returned correctly diff --git a/backend/windmill-dep-map/src/scoped_dependency_map.rs b/backend/windmill-dep-map/src/scoped_dependency_map.rs index 9821b8e610..fcee6fe5f1 100644 --- a/backend/windmill-dep-map/src/scoped_dependency_map.rs +++ b/backend/windmill-dep-map/src/scoped_dependency_map.rs @@ -445,7 +445,28 @@ SELECT importer_node_id, imported_path, imported_lockfile_hash } } - /// Get dependents of any imported path - returns scripts/flows/apps that depend on it + /// Get imports of a given importer path - returns paths that the importer depends on + pub async fn get_imports<'c>( + importer_path: &str, + workspace_id: &str, + e: impl PgExecutor<'c>, + ) -> Result> { + sqlx::query_scalar!( + r#" + SELECT DISTINCT imported_path as "imported_path!" + FROM dependency_map + WHERE workspace_id = $1 + AND importer_path = $2 + AND imported_path NOT LIKE 'dependencies/%' + "#, + workspace_id, + importer_path + ) + .fetch_all(e) + .await + .map_err(Error::from) + } + pub async fn get_dependents<'c>( imported_path: &str, workspace_id: &str, diff --git a/backend/windmill-dep-map/src/trigger_dependents.rs b/backend/windmill-dep-map/src/trigger_dependents.rs index f17d3863ac..1254e41ed1 100644 --- a/backend/windmill-dep-map/src/trigger_dependents.rs +++ b/backend/windmill-dep-map/src/trigger_dependents.rs @@ -61,7 +61,7 @@ pub async fn trigger_dependents_to_recompute_dependencies( ); let mut debouncing_settings = DebouncingSettings { - debounce_key: Some(format!("{w_id}:{importer_path}:dependency")), + debounce_key: Some(format!("{w_id}:{importer_path}:{importer_kind}:dependency")), debounce_delay_s: Some(5), ..Default::default() }; diff --git a/backend/windmill-native-triggers/src/handler.rs b/backend/windmill-native-triggers/src/handler.rs index cd6e1a8100..60eb006a97 100644 --- a/backend/windmill-native-triggers/src/handler.rs +++ b/backend/windmill-native-triggers/src/handler.rs @@ -513,13 +513,13 @@ pub fn service_routes(handler: T) -> Router { let standard_routes = Router::new() .route("/create", post(create_native_trigger::)) .route("/list", get(list_native_triggers_handler::)) - .route("/get/:external_id", get(get_native_trigger_handler::)) + .route("/get/{external_id}", get(get_native_trigger_handler::)) .route( - "/update/:external_id", + "/update/{external_id}", post(update_native_trigger_handler::), ) .route( - "/delete/:external_id", + "/delete/{external_id}", delete(delete_native_trigger_handler::), ); diff --git a/backend/windmill-native-triggers/src/workspace_integrations.rs b/backend/windmill-native-triggers/src/workspace_integrations.rs index 87d40d5b05..9453f033ab 100644 --- a/backend/windmill-native-triggers/src/workspace_integrations.rs +++ b/backend/windmill-native-triggers/src/workspace_integrations.rs @@ -964,22 +964,22 @@ async fn generate_instance_connect_url( pub fn workspaced_service() -> Router { let router = Router::new() .route("/list", get(list_integrations)) - .route("/:service_name/exists", get(integration_exist)) - .route("/:service_name/create", post(create_workspace_integration)) + .route("/{service_name}/exists", get(integration_exist)) + .route("/{service_name}/create", post(create_workspace_integration)) .route( - "/:service_name/generate_connect_url", + "/{service_name}/generate_connect_url", post(generate_connect_url), ) .route( - "/:service_name/instance_sharing_available", + "/{service_name}/instance_sharing_available", get(check_instance_sharing_available), ) .route( - "/:service_name/generate_instance_connect_url", + "/{service_name}/generate_instance_connect_url", post(generate_instance_connect_url), ) - .route("/:service_name/delete", delete(delete_integration)) - .route("/:service_name/callback", post(oauth_callback)); + .route("/{service_name}/delete", delete(delete_integration)) + .route("/{service_name}/callback", post(oauth_callback)); Router::new().nest("/integrations", router) } diff --git a/backend/windmill-oauth/src/lib.rs b/backend/windmill-oauth/src/lib.rs index d614837237..eff53be481 100644 --- a/backend/windmill-oauth/src/lib.rs +++ b/backend/windmill-oauth/src/lib.rs @@ -516,7 +516,7 @@ pub async fn exchange_code( }; let csrf_state = cookies .get(name) - .map(|x| x.value().to_string()) + .map(|x| x.value_trimmed().to_string()) .unwrap_or("".to_string()); if callback.state != csrf_state { return Err(error::Error::BadRequest("csrf did not match".to_string())); diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index 4caf67f348..e8d33da2e2 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -5,13 +5,13 @@ use std::collections::HashMap; use quick_cache::sync::Cache; use windmill_common::error::{self}; +#[cfg(feature = "parquet")] +use async_trait::async_trait; #[cfg(feature = "parquet")] use aws_config::{default_provider::credentials::DefaultCredentialsChain, Region}; #[cfg(feature = "parquet")] use aws_sdk_sts::config::ProvideCredentials; #[cfg(feature = "parquet")] -use axum::async_trait; -#[cfg(feature = "parquet")] use bytes::Bytes; #[cfg(feature = "parquet")] use chrono::{DateTime, Utc}; diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index a6ba745c24..53c7460c4e 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -77,8 +77,8 @@ use windmill_common::{ users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL}, utils::{not_found_if_none, report_critical_error, StripPath, WarnAfterExt}, worker::{ - to_raw_value, CLOUD_HOSTED, DISABLE_FLOW_SCRIPT, NO_LOGS, WORKER_PULL_QUERIES, - WORKER_SUSPENDED_PULL_QUERY, + to_raw_value, CLOUD_HOSTED, DISABLE_FLOW_SCRIPT, NO_LOGS, PREVIEW_TAGS_OVERRIDE, + WORKER_PULL_QUERIES, WORKER_SUSPENDED_PULL_QUERY, }, DB, METRICS_ENABLED, }; @@ -3302,7 +3302,10 @@ pub async fn pull( }; if let Some(job) = job.as_ref() { - if job.is_flow() || job.is_dependency() { + if (job.is_flow() || job.is_dependency()) + && !(job.kind.is_preview() + && PREVIEW_TAGS_OVERRIDE.load(std::sync::atomic::Ordering::Relaxed)) + { let per_workspace = per_workspace_tag(&job.workspace_id).await; let base_tag = if job.is_flow() { "flow".to_string() @@ -5493,25 +5496,35 @@ async fn push_inner<'c, 'd>( }; interpolated_tag.unwrap_or_else(|| { - language - .as_ref() - .map(|x| { - let tag_lang = if x == &ScriptLang::Bunnative { - if job_kind == JobKind::Dependencies { - ScriptLang::Bun.as_str() + if job_kind.is_preview() + && PREVIEW_TAGS_OVERRIDE.load(std::sync::atomic::Ordering::Relaxed) + { + if per_workspace { + format!("preview-{}", workspace_id) + } else { + "preview".to_string() + } + } else { + language + .as_ref() + .map(|x| { + let tag_lang = if x == &ScriptLang::Bunnative { + if job_kind == JobKind::Dependencies { + ScriptLang::Bun.as_str() + } else { + ScriptLang::Nativets.as_str() + } } else { - ScriptLang::Nativets.as_str() + x.as_str() + }; + if per_workspace { + format!("{}-{}", tag_lang, workspace_id) + } else { + tag_lang.to_string() } - } else { - x.as_str() - }; - if per_workspace { - format!("{}-{}", tag_lang, workspace_id) - } else { - tag_lang.to_string() - } - }) - .unwrap_or_else(default) + }) + .unwrap_or_else(default) + } }) }; diff --git a/backend/windmill-runtime-nativets/src/windmill-client.js b/backend/windmill-runtime-nativets/src/windmill-client.js index 24e781343e..b0816dd78c 100644 --- a/backend/windmill-runtime-nativets/src/windmill-client.js +++ b/backend/windmill-runtime-nativets/src/windmill-client.js @@ -3000,6 +3000,7 @@ var $RawScript = { "nativets", "duckdb", "ruby", + "rlang", // for related places search: ADD_NEW_LANG ], }, diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index 102bab8585..8fb77626e5 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -7,6 +7,7 @@ */ use std::collections::HashMap; +use std::net::IpAddr; use windmill_api_auth::{ check_scopes, maybe_refresh_folders, require_owner_of_path, require_super_admin, ApiAuthed, @@ -55,26 +56,26 @@ pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_resources)) .route("/list_search", get(list_search_resources)) - .route("/list_names/:type", get(list_names)) - .route("/get/*path", get(get_resource)) - .route("/exists/*path", get(exists_resource)) - .route("/get_value/*path", get(get_resource_value)) + .route("/list_names/{type}", get(list_names)) + .route("/get/{*path}", get(get_resource)) + .route("/exists/{*path}", get(exists_resource)) + .route("/get_value/{*path}", get(get_resource_value)) .route( - "/get_value_interpolated/*path", + "/get_value_interpolated/{*path}", get(get_resource_value_interpolated), ) - .route("/update/*path", post(update_resource)) - .route("/update_value/*path", post(update_resource_value)) - .route("/delete/*path", delete(delete_resource)) + .route("/update/{*path}", post(update_resource)) + .route("/update_value/{*path}", post(update_resource_value)) + .route("/delete/{*path}", delete(delete_resource)) .route("/delete_bulk", delete(delete_resources_bulk)) .route("/create", post(create_resource)) - .route("/git_commit_hash/*path", get(get_git_commit_hash)) + .route("/git_commit_hash/{*path}", get(get_git_commit_hash)) .route("/type/list", get(list_resource_types)) .route("/type/listnames", get(list_resource_types_names)) - .route("/type/get/:name", get(get_resource_type)) - .route("/type/exists/:name", get(exists_resource_type)) - .route("/type/update/:name", post(update_resource_type)) - .route("/type/delete/:name", delete(delete_resource_type)) + .route("/type/get/{name}", get(get_resource_type)) + .route("/type/exists/{name}", get(exists_resource_type)) + .route("/type/update/{name}", post(update_resource_type)) + .route("/type/delete/{name}", delete(delete_resource_type)) .route( "/file_resource_type_to_file_ext_map", get(file_resource_ext_to_resource_type), @@ -83,7 +84,7 @@ pub fn workspaced_service() -> Router { } pub fn public_service() -> Router { - Router::new().route("/custom_component/:name", get(custom_component)) + Router::new().route("/custom_component/{name}", get(custom_component)) } #[derive(FromRow, Serialize, Deserialize)] @@ -297,7 +298,11 @@ async fn list_resources( } if let Some(value) = &lq.value { - sqlb.and_where("resource.value @> ?".bind(&value.replace("'", "''"))); + if let Ok(v) = serde_json::from_str::(value) { + sqlb.and_where("resource.value @> ?".bind(&v.to_string())); + } else { + sqlb.and_where("FALSE"); + } } if let Some(broad_filter) = &lq.broad_filter { @@ -570,6 +575,16 @@ pub async fn transform_json_value( .await?; Ok(Value::String(v)) } + Value::String(y) if y.starts_with("$jsonvar:") => { + let path = y.strip_prefix("$jsonvar:").unwrap(); + + let v = + crate::variables::get_value_internal(&db_with_opt_authed, workspace, path, false) + .await?; + serde_json::from_str::(&v).map_err(|e| { + Error::internal_err(format!("Failed to parse $jsonvar value as JSON: {e}")) + }) + } Value::String(y) if y.starts_with("$res:") => { let path = y.strip_prefix("$res:").unwrap(); if path.split("/").count() < 2 { @@ -1767,9 +1782,74 @@ struct GitRepositoryResource { branch: Option, } -/// Validates a git URL to prevent git option injection attacks. -/// Git URLs starting with '-' could be interpreted as command-line options. -fn validate_git_url(url: &str) -> Result<()> { +/// Checks whether an IP address belongs to a private, loopback, link-local, or +/// otherwise reserved range that should not be reachable from git operations. +fn is_private_or_reserved_ip(ip: &IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => { + v4.is_loopback() + || v4.is_private() + || v4.is_link_local() + || v4.is_unspecified() + || v4.is_broadcast() + // 100.64.0.0/10 (Carrier-grade NAT / CGNAT) + || (v4.octets()[0] == 100 && (v4.octets()[1] & 0xC0) == 64) + } + IpAddr::V6(v6) => { + v6.is_loopback() + || v6.is_unspecified() + // IPv4-mapped IPv6 (::ffff:x.x.x.x) — check the inner v4 + || v6.to_ipv4_mapped().map_or(false, |v4| { + is_private_or_reserved_ip(&IpAddr::V4(v4)) + }) + } + } +} + +/// Extracts the hostname from a git URL. +/// +/// Handles standard URLs (`https://host/path`, `ssh://user@host/path`) and +/// SCP-style (`user@host:path`). +fn extract_host_from_git_url(url: &str) -> Option { + if let Some(after_scheme) = url.split("://").nth(1) { + // Standard URL with scheme + let host_part = match after_scheme.find('@') { + Some(pos) => &after_scheme[pos + 1..], + None => after_scheme, + }; + // Handle IPv6 in brackets: [::1] + if host_part.starts_with('[') { + let end = host_part.find(']')?; + let host = &host_part[1..end]; + return if host.is_empty() { + None + } else { + Some(host.to_lowercase()) + }; + } + let host_port = host_part.split('/').next()?; + let host = host_port.rsplit_once(':').map_or(host_port, |(h, _)| h); + if host.is_empty() { + return None; + } + return Some(host.to_lowercase()); + } + + // SCP-style: user@host:path + if let Some(at_pos) = url.find('@') { + let after_at = &url[at_pos + 1..]; + let host = after_at.split(':').next()?; + if host.is_empty() { + return None; + } + return Some(host.to_lowercase()); + } + + None +} + +/// Validates a git URL to prevent option injection, SSRF, and local file read. +async fn validate_git_url(url: &str) -> Result<()> { let url = url.trim(); if url.is_empty() { return Err(Error::BadRequest("Git URL cannot be empty".to_string())); @@ -1779,12 +1859,59 @@ fn validate_git_url(url: &str) -> Result<()> { "Git URL cannot start with '-' (potential option injection)".to_string(), )); } - // Block other potentially dangerous patterns if url.contains('\0') || url.contains('\n') || url.contains('\r') { return Err(Error::BadRequest( "Git URL contains invalid characters".to_string(), )); } + + let lower = url.to_lowercase(); + + // Allowlist of URL formats — blocks file://, ftp://, local paths, etc. + let has_valid_scheme = lower.starts_with("https://") + || lower.starts_with("http://") + || lower.starts_with("git://") + || lower.starts_with("ssh://"); + + // SCP-style: user@host:path (no scheme, has @ before :) + let is_scp_style = !url.contains("://") && url.contains('@') && url.contains(':'); + + if !has_valid_scheme && !is_scp_style { + return Err(Error::BadRequest( + "Git URL must use https://, http://, git://, ssh://, or user@host:path format" + .to_string(), + )); + } + + let host = extract_host_from_git_url(url) + .ok_or_else(|| Error::BadRequest("Could not parse hostname from git URL".to_string()))?; + + if host == "localhost" || host.ends_with(".local") || host == "[::1]" { + return Err(Error::BadRequest( + "Git URLs targeting localhost or local network are not allowed".to_string(), + )); + } + + // Check literal IP addresses + if let Ok(ip) = host.parse::() { + if is_private_or_reserved_ip(&ip) { + return Err(Error::BadRequest( + "Git URLs targeting private or reserved IP addresses are not allowed".to_string(), + )); + } + } else { + // Hostname — resolve via DNS and reject if any address is private + if let Ok(addrs) = tokio::net::lookup_host(format!("{}:443", host)).await { + for addr in addrs { + if is_private_or_reserved_ip(&addr.ip()) { + return Err(Error::BadRequest( + "Git URL hostname resolves to a private or reserved IP address".to_string(), + )); + } + } + } + } + Ok(()) } @@ -1976,8 +2103,8 @@ async fn get_repo_latest_commit_hash( git_resource: &GitRepositoryResource, git_ssh_command: Option, ) -> Result { - // Validate URL and branch to prevent option injection attacks - validate_git_url(&git_resource.url)?; + // Validate URL and branch to prevent option injection and SSRF attacks + validate_git_url(&git_resource.url).await?; let ref_spec = git_resource .branch @@ -2141,4 +2268,149 @@ mod tests { assert!(result.is_err()); } + + #[test] + fn test_extract_host_from_git_url() { + // Standard HTTPS + assert_eq!( + extract_host_from_git_url("https://github.com/user/repo.git"), + Some("github.com".to_string()) + ); + // HTTPS with port + assert_eq!( + extract_host_from_git_url("https://git.example.com:8443/repo.git"), + Some("git.example.com".to_string()) + ); + // SSH with scheme + assert_eq!( + extract_host_from_git_url("ssh://git@github.com/user/repo.git"), + Some("github.com".to_string()) + ); + // SCP-style + assert_eq!( + extract_host_from_git_url("git@github.com:user/repo.git"), + Some("github.com".to_string()) + ); + // Git protocol + assert_eq!( + extract_host_from_git_url("git://example.com/repo.git"), + Some("example.com".to_string()) + ); + // IPv6 in brackets + assert_eq!( + extract_host_from_git_url("http://[::1]:8080/repo.git"), + Some("::1".to_string()) + ); + // No host extractable + assert_eq!(extract_host_from_git_url("/local/path"), None); + assert_eq!( + extract_host_from_git_url("file:///etc/passwd"), + Some("".to_string()).filter(|s| !s.is_empty()) + ); + } + + #[test] + fn test_is_private_or_reserved_ip() { + use std::net::IpAddr; + // Loopback + assert!(is_private_or_reserved_ip( + &"127.0.0.1".parse::().unwrap() + )); + assert!(is_private_or_reserved_ip( + &"127.0.0.2".parse::().unwrap() + )); + // Private ranges + assert!(is_private_or_reserved_ip( + &"10.0.0.1".parse::().unwrap() + )); + assert!(is_private_or_reserved_ip( + &"172.16.0.1".parse::().unwrap() + )); + assert!(is_private_or_reserved_ip( + &"192.168.1.1".parse::().unwrap() + )); + // Link-local / cloud metadata + assert!(is_private_or_reserved_ip( + &"169.254.169.254".parse::().unwrap() + )); + // CGNAT + assert!(is_private_or_reserved_ip( + &"100.64.0.1".parse::().unwrap() + )); + // Unspecified + assert!(is_private_or_reserved_ip( + &"0.0.0.0".parse::().unwrap() + )); + // IPv6 loopback + assert!(is_private_or_reserved_ip(&"::1".parse::().unwrap())); + // IPv4-mapped IPv6 + assert!(is_private_or_reserved_ip( + &"::ffff:127.0.0.1".parse::().unwrap() + )); + // Public IPs should pass + assert!(!is_private_or_reserved_ip( + &"8.8.8.8".parse::().unwrap() + )); + assert!(!is_private_or_reserved_ip( + &"140.82.121.4".parse::().unwrap() + )); + } + + #[tokio::test] + async fn test_validate_git_url_blocks_file_scheme() { + let result = validate_git_url("file:///etc/passwd").await; + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("https://")); + } + + #[tokio::test] + async fn test_validate_git_url_blocks_private_ips() { + assert!(validate_git_url("http://127.0.0.1/repo.git").await.is_err()); + assert!(validate_git_url("http://169.254.169.254/latest/meta-data/") + .await + .is_err()); + assert!(validate_git_url("http://10.0.0.1/repo.git").await.is_err()); + assert!(validate_git_url("http://172.16.0.1/repo.git") + .await + .is_err()); + assert!(validate_git_url("http://192.168.1.1/repo.git") + .await + .is_err()); + assert!(validate_git_url("git://0.0.0.0/repo.git").await.is_err()); + } + + #[tokio::test] + async fn test_validate_git_url_blocks_localhost() { + assert!(validate_git_url("http://localhost/repo.git").await.is_err()); + assert!(validate_git_url("http://myhost.local/repo.git") + .await + .is_err()); + } + + #[tokio::test] + async fn test_validate_git_url_blocks_local_paths() { + assert!(validate_git_url("/etc/passwd").await.is_err()); + assert!(validate_git_url("../relative/path").await.is_err()); + assert!(validate_git_url("./local/repo").await.is_err()); + } + + #[tokio::test] + async fn test_validate_git_url_allows_valid_urls() { + // These should succeed (host resolution may fail but validation passes) + assert!(validate_git_url("https://github.com/user/repo.git") + .await + .is_ok()); + assert!(validate_git_url("git@github.com:user/repo.git") + .await + .is_ok()); + assert!(validate_git_url("ssh://git@github.com/user/repo.git") + .await + .is_ok()); + } + + #[tokio::test] + async fn test_validate_git_url_blocks_option_injection() { + assert!(validate_git_url("-evil").await.is_err()); + assert!(validate_git_url("--upload-pack=evil").await.is_err()); + } } diff --git a/backend/windmill-store/src/variables.rs b/backend/windmill-store/src/variables.rs index 964f81809c..c893f61cb0 100644 --- a/backend/windmill-store/src/variables.rs +++ b/backend/windmill-store/src/variables.rs @@ -54,11 +54,11 @@ pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_variables)) .route("/list_contextual", get(list_contextual_variables)) - .route("/get/*path", get(get_variable)) - .route("/get_value/*path", get(get_value)) - .route("/exists/*path", get(exists_variable)) - .route("/update/*path", post(update_variable)) - .route("/delete/*path", delete(delete_variable)) + .route("/get/{*path}", get(get_variable)) + .route("/get_value/{*path}", get(get_value)) + .route("/exists/{*path}", get(exists_variable)) + .route("/update/{*path}", post(update_variable)) + .route("/delete/{*path}", delete(delete_variable)) .route("/delete_bulk", delete(delete_variables_bulk)) .route("/create", post(create_variable)) .route("/encrypt", post(encrypt_value)) diff --git a/backend/windmill-test-utils/Cargo.toml b/backend/windmill-test-utils/Cargo.toml index d1729a7511..a221e53911 100644 --- a/backend/windmill-test-utils/Cargo.toml +++ b/backend/windmill-test-utils/Cargo.toml @@ -14,6 +14,7 @@ private = ["windmill-api/private"] enterprise = ["windmill-api/enterprise"] python = ["windmill-common/python"] deno_core = ["dep:windmill-runtime-nativets"] +mcp = ["windmill-api/mcp"] agent_worker_server = ["dep:windmill-api-agent-workers"] run_inline = ["windmill-api/run_inline"] duckdb = ["windmill-worker/duckdb"] @@ -35,5 +36,6 @@ tokio.workspace = true uuid.workspace = true chrono.workspace = true axum.workspace = true +async-trait.workspace = true anyhow.workspace = true tracing.workspace = true diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index d9e22eaa84..153bdeeb7c 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -81,20 +81,29 @@ pub struct ApiServer { impl ApiServer { pub async fn start(db: Pool) -> anyhow::Result { - Self::start_inner(db, false).await + Self::start_inner(db, false, false).await } pub async fn start_agent_mode(db: Pool) -> anyhow::Result { - Self::start_inner(db, true).await + Self::start_inner(db, true, false).await } /// Start the API server with server_mode=true so trigger listeners are active. /// Alias for `start_agent_mode` with a clearer name for trigger e2e tests. pub async fn start_with_listeners(db: Pool) -> anyhow::Result { - Self::start_inner(db, true).await + Self::start_inner(db, true, false).await } - async fn start_inner(db: Pool, agent_mode: bool) -> anyhow::Result { + /// Start the API server with mcp_mode=true so MCP routes are active. + pub async fn start_mcp(db: Pool) -> anyhow::Result { + Self::start_inner(db, false, true).await + } + + async fn start_inner( + db: Pool, + server_mode: bool, + mcp_mode: bool, + ) -> anyhow::Result { let (tx, rx) = tokio::sync::broadcast::channel::<()>(1); let listener = tokio::net::TcpListener::bind("127.0.0.1:0") @@ -114,8 +123,8 @@ impl ApiServer { listener, rx, port_tx, - agent_mode, - false, + server_mode, + mcp_mode, format!("http://localhost:{}", addr.port()), Some(name.clone()), )); @@ -478,7 +487,7 @@ pub async fn completed_job(uuid: Uuid, db: &Pool) -> CompletedJob { .unwrap() } -#[axum::async_trait(?Send)] +#[async_trait::async_trait(?Send)] pub trait StreamFind: futures::Stream + Unpin + Sized { async fn find(self, item: &Self::Item) -> Option where diff --git a/backend/windmill-trigger-email/src/handler_oss.rs b/backend/windmill-trigger-email/src/handler_oss.rs index 9579bee274..b6cac94cc2 100644 --- a/backend/windmill-trigger-email/src/handler_oss.rs +++ b/backend/windmill-trigger-email/src/handler_oss.rs @@ -8,7 +8,7 @@ pub use super::handler_ee::*; #[cfg(not(feature = "private"))] use { super::EmailTrigger, - axum::async_trait, + async_trait::async_trait, sqlx::PgConnection, windmill_api_auth::ApiAuthed, windmill_common::{ diff --git a/backend/windmill-trigger-gcp/src/handler_oss.rs b/backend/windmill-trigger-gcp/src/handler_oss.rs index b259c87834..5cf0f17c02 100644 --- a/backend/windmill-trigger-gcp/src/handler_oss.rs +++ b/backend/windmill-trigger-gcp/src/handler_oss.rs @@ -5,7 +5,7 @@ pub use super::handler_ee::*; #[cfg(not(feature = "private"))] use { super::GcpTrigger, - axum::async_trait, + async_trait::async_trait, sqlx::PgConnection, windmill_api_auth::ApiAuthed, windmill_common::{ diff --git a/backend/windmill-trigger-http/Cargo.toml b/backend/windmill-trigger-http/Cargo.toml index 865b82a682..9e81e1d0dd 100644 --- a/backend/windmill-trigger-http/Cargo.toml +++ b/backend/windmill-trigger-http/Cargo.toml @@ -45,3 +45,4 @@ itertools.workspace = true thiserror.workspace = true anyhow.workspace = true hex.workspace = true +chrono.workspace = true diff --git a/backend/windmill-trigger-http/src/handler.rs b/backend/windmill-trigger-http/src/handler.rs index f6ca4739da..46e278ca5b 100644 --- a/backend/windmill-trigger-http/src/handler.rs +++ b/backend/windmill-trigger-http/src/handler.rs @@ -2,7 +2,8 @@ use super::{ validate_authentication_method, HttpConfig, HttpConfigRequest, HttpMethod, HttpTrigger, RouteExists, ROUTE_PATH_KEY_RE, VALID_ROUTE_PATH_RE, }; -use axum::{async_trait, extract::Path, routing::post, Extension, Json, Router}; +use async_trait::async_trait; +use axum::{extract::Path, routing::post, Extension, Json, Router}; use http::StatusCode; use sqlx::PgConnection; use std::collections::HashSet; diff --git a/backend/windmill-trigger-http/src/http_trigger_auth.rs b/backend/windmill-trigger-http/src/http_trigger_auth.rs index 987d0ac811..19766cdbc2 100644 --- a/backend/windmill-trigger-http/src/http_trigger_auth.rs +++ b/backend/windmill-trigger-http/src/http_trigger_auth.rs @@ -12,6 +12,23 @@ use sha1::Sha1; use sha2::{Sha256, Sha512}; use std::{borrow::Cow, collections::HashMap}; +const MAX_TIMESTAMP_AGE_SECS: i64 = 300; // 5 minutes + +fn validate_unix_timestamp(timestamp_str: &str) -> Result<(), AuthenticationError> { + let ts: i64 = timestamp_str + .parse() + .map_err(|_| AuthenticationError::InvalidTimestamp)?; + let now = chrono::Utc::now().timestamp(); + let diff = now - ts; + if diff > MAX_TIMESTAMP_AGE_SECS { + return Err(AuthenticationError::TimestampTooOldError); + } + if diff < -MAX_TIMESTAMP_AGE_SECS { + return Err(AuthenticationError::FutureTimestampError); + } + Ok(()) +} + pub type HmacSha256 = Hmac; pub type HmacSha512 = Hmac; pub type HmacSha1 = Hmac; @@ -82,6 +99,11 @@ mod slack { SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), )) } + + fn validate_timestamp(&self, headers: &HeaderMap) -> Result<(), AuthenticationError> { + let ts = headers.try_get_webhook_header("X-Slack-Request-Timestamp")?; + validate_unix_timestamp(ts) + } } } @@ -126,6 +148,13 @@ mod stripe { SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), )) } + + fn validate_timestamp(&self, headers: &HeaderMap) -> Result<(), AuthenticationError> { + let sig_header = headers.try_get_webhook_header("STRIPE-SIGNATURE")?; + let sig = parse_signature(sig_header, (",", "=")); + let ts = *sig.get("t").ok_or(AuthenticationError::InvalidTimestamp)?; + validate_unix_timestamp(ts) + } } } @@ -170,6 +199,13 @@ mod tiktok { SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), )) } + + fn validate_timestamp(&self, headers: &HeaderMap) -> Result<(), AuthenticationError> { + let sig_header = headers.try_get_webhook_header("TikTok-Signature")?; + let sig = parse_signature(sig_header, (",", "=")); + let ts = *sig.get("t").ok_or(AuthenticationError::InvalidTimestamp)?; + validate_unix_timestamp(ts) + } } } @@ -244,6 +280,23 @@ mod twitch { Ok(Some(response.into_response())) } + + fn validate_timestamp(&self, headers: &HeaderMap) -> Result<(), AuthenticationError> { + let ts_str = headers.try_get_webhook_header("Twitch-Eventsub-Message-Timestamp")?; + let ts: chrono::DateTime = chrono::DateTime::parse_from_rfc3339(ts_str) + .map_err(|_| AuthenticationError::InvalidTimestamp)? + .into(); + let now = chrono::Utc::now(); + let diff = (now - ts).num_seconds(); + // Twitch recommends 10 minutes tolerance + if diff > 600 { + return Err(AuthenticationError::TimestampTooOldError); + } + if diff < -600 { + return Err(AuthenticationError::FutureTimestampError); + } + Ok(()) + } } } @@ -321,6 +374,11 @@ mod zoom { SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), )) } + + fn validate_timestamp(&self, headers: &HeaderMap) -> Result<(), AuthenticationError> { + let ts = headers.try_get_webhook_header("x-zm-request-timestamp")?; + validate_unix_timestamp(ts) + } } } @@ -397,6 +455,10 @@ pub trait WebhookHandler { headers: &'header HeaderMap, raw_payload: &'payload str, ) -> Result, AuthenticationError>; + + fn validate_timestamp(&self, _headers: &HeaderMap) -> Result<(), AuthenticationError> { + Ok(()) + } } #[derive(Clone, Copy, Debug, Serialize, Deserialize)] @@ -592,6 +654,10 @@ impl AuthenticationMethod { return Ok(Some(challenge_response)); } + if let Some(handler) = handler { + handler.validate_timestamp(headers)?; + } + let authentication_data = match handler { Some(handler) => handler.get_hmac_authentication_data(headers, raw_payload)?, None => { @@ -621,7 +687,7 @@ impl AuthenticationMethod { let api_key_to_cmp = headers .try_get_webhook_header(&api_key_header) .map_err(|_| AuthenticationError::InvalidApiKey)?; - if api_key_to_cmp != api_key_secret { + if !constant_time_eq(api_key_to_cmp.as_bytes(), api_key_secret.as_bytes()) { return Err(AuthenticationError::InvalidApiKey); } } @@ -654,8 +720,11 @@ impl AuthenticationMethod { return Err(AuthenticationError::UnauthorizedBasicHttpAuth); } - if credentials.get(0).unwrap() != username - || credentials.get(1).unwrap() != password + if !constant_time_eq(credentials.get(0).unwrap().as_bytes(), username.as_bytes()) + || !constant_time_eq( + credentials.get(1).unwrap().as_bytes(), + password.as_bytes(), + ) { return Err(AuthenticationError::UnauthorizedBasicHttpAuth); } @@ -667,7 +736,6 @@ impl AuthenticationMethod { } #[derive(thiserror::Error, Debug)] -#[allow(unused)] pub enum AuthenticationError { #[error("failed to parse timestamp")] InvalidTimestamp, @@ -1207,11 +1275,15 @@ mod tests { headers } + fn current_timestamp() -> String { + chrono::Utc::now().timestamp().to_string() + } + #[test] fn test_slack_authenticate_valid() { let secret = "slack_signing_secret"; let payload = "token=xxx&command=%2Ftest".to_string(); - let timestamp = "1531420618"; + let timestamp = ¤t_timestamp(); let headers = slack_headers(secret, &payload, timestamp); let method = AuthenticationMethod::Signature(SignatureAuthentication { @@ -1225,7 +1297,7 @@ mod tests { } #[test] - fn test_slack_authenticate_wrong_timestamp() { + fn test_slack_authenticate_stale_timestamp_rejected() { let secret = "slack_secret"; let payload = "data".to_string(); let headers = slack_headers(secret, &payload, "1000000000"); @@ -1235,10 +1307,10 @@ mod tests { secret_key: secret.to_string(), authentication_config: None, }); - // Constructed with timestamp "1000000000" but that's valid - it just needs to match - assert!(method - .authenticate_http_request(&headers, Some(&payload)) - .is_ok()); + assert!(matches!( + method.authenticate_http_request(&headers, Some(&payload)), + Err(AuthenticationError::TimestampTooOldError) + )); } // --- Stripe webhook end-to-end --- @@ -1259,7 +1331,7 @@ mod tests { fn test_stripe_authenticate_valid() { let secret = "whsec_stripe_secret"; let payload = r#"{"id":"evt_123"}"#.to_string(); - let timestamp = "1614556800"; + let timestamp = ¤t_timestamp(); let headers = stripe_headers(secret, &payload, timestamp); let method = AuthenticationMethod::Signature(SignatureAuthentication { @@ -1305,7 +1377,7 @@ mod tests { fn test_tiktok_authenticate_valid() { let secret = "tiktok_secret"; let payload = r#"{"event":"video.upload"}"#.to_string(); - let timestamp = "1700000000"; + let timestamp = ¤t_timestamp(); let headers = tiktok_headers(secret, &payload, timestamp); let method = AuthenticationMethod::Signature(SignatureAuthentication { @@ -1350,17 +1422,16 @@ mod tests { headers } + fn current_rfc3339_timestamp() -> String { + chrono::Utc::now().to_rfc3339() + } + #[test] fn test_twitch_authenticate_valid_notification() { let secret = "twitch_secret"; let payload = r#"{"subscription":{},"event":{"user_id":"123"}}"#.to_string(); - let headers = twitch_headers( - secret, - &payload, - "msg-123", - "2024-01-01T00:00:00Z", - "notification", - ); + let ts = current_rfc3339_timestamp(); + let headers = twitch_headers(secret, &payload, "msg-123", &ts, "notification"); let method = AuthenticationMethod::Signature(SignatureAuthentication { signature_provider: WebhookType::Twitch, @@ -1376,11 +1447,12 @@ mod tests { fn test_twitch_challenge_response() { let secret = "twitch_secret"; let payload = r#"{"challenge":"test_challenge_string","subscription":{"id":"sub-123"}}"#; + let ts = current_rfc3339_timestamp(); let headers = twitch_headers( secret, payload, "msg-456", - "2024-01-01T00:00:00Z", + &ts, "webhook_callback_verification", ); @@ -1396,13 +1468,8 @@ mod tests { fn test_twitch_non_challenge_returns_none() { let secret = "twitch_secret"; let payload = r#"{"subscription":{},"event":{}}"#; - let headers = twitch_headers( - secret, - payload, - "msg-789", - "2024-01-01T00:00:00Z", - "notification", - ); + let ts = current_rfc3339_timestamp(); + let headers = twitch_headers(secret, payload, "msg-789", &ts, "notification"); let handler = WebhookType::Twitch.get_webhook_handler().unwrap(); let config_data = SignatureConfigData { secret_key: secret }; @@ -1434,7 +1501,7 @@ mod tests { fn test_zoom_authenticate_valid() { let secret = "zoom_secret"; let payload = r#"{"event":"meeting.started"}"#.to_string(); - let timestamp = "1700000000"; + let timestamp = ¤t_timestamp(); let headers = zoom_headers(secret, &payload, timestamp); let method = AuthenticationMethod::Signature(SignatureAuthentication { @@ -1789,4 +1856,84 @@ mod tests { let response = AuthenticationError::InvalidTimestamp.into_response(); assert_eq!(response.status(), StatusCode::BAD_REQUEST); } + + // --- validate_unix_timestamp --- + + #[test] + fn test_validate_unix_timestamp_current() { + let now = chrono::Utc::now().timestamp().to_string(); + assert!(validate_unix_timestamp(&now).is_ok()); + } + + #[test] + fn test_validate_unix_timestamp_recent() { + let ts = (chrono::Utc::now().timestamp() - 60).to_string(); + assert!(validate_unix_timestamp(&ts).is_ok()); + } + + #[test] + fn test_validate_unix_timestamp_too_old() { + let ts = (chrono::Utc::now().timestamp() - 600).to_string(); + assert!(matches!( + validate_unix_timestamp(&ts), + Err(AuthenticationError::TimestampTooOldError) + )); + } + + #[test] + fn test_validate_unix_timestamp_future() { + let ts = (chrono::Utc::now().timestamp() + 600).to_string(); + assert!(matches!( + validate_unix_timestamp(&ts), + Err(AuthenticationError::FutureTimestampError) + )); + } + + #[test] + fn test_validate_unix_timestamp_invalid() { + assert!(matches!( + validate_unix_timestamp("not-a-number"), + Err(AuthenticationError::InvalidTimestamp) + )); + } + + // --- Slack timestamp validation --- + + #[test] + fn test_slack_validate_timestamp_current() { + let handler = slack::Slack; + let mut headers = HeaderMap::new(); + let now = chrono::Utc::now().timestamp().to_string(); + headers.insert("X-Slack-Request-Timestamp", now.parse().unwrap()); + assert!(handler.validate_timestamp(&headers).is_ok()); + } + + #[test] + fn test_slack_validate_timestamp_stale() { + let handler = slack::Slack; + let mut headers = HeaderMap::new(); + let old = (chrono::Utc::now().timestamp() - 600).to_string(); + headers.insert("X-Slack-Request-Timestamp", old.parse().unwrap()); + assert!(handler.validate_timestamp(&headers).is_err()); + } + + // --- Twitch timestamp validation (ISO 8601) --- + + #[test] + fn test_twitch_validate_timestamp_current() { + let handler = twitch::Twitch; + let mut headers = HeaderMap::new(); + let now = chrono::Utc::now().to_rfc3339(); + headers.insert("Twitch-Eventsub-Message-Timestamp", now.parse().unwrap()); + assert!(handler.validate_timestamp(&headers).is_ok()); + } + + #[test] + fn test_twitch_validate_timestamp_stale() { + let handler = twitch::Twitch; + let mut headers = HeaderMap::new(); + let old = (chrono::Utc::now() - chrono::TimeDelta::seconds(1200)).to_rfc3339(); + headers.insert("Twitch-Eventsub-Message-Timestamp", old.parse().unwrap()); + assert!(handler.validate_timestamp(&headers).is_err()); + } } diff --git a/backend/windmill-trigger-kafka/src/handler_oss.rs b/backend/windmill-trigger-kafka/src/handler_oss.rs index ace4b87d0a..2e1cb8bfb4 100644 --- a/backend/windmill-trigger-kafka/src/handler_oss.rs +++ b/backend/windmill-trigger-kafka/src/handler_oss.rs @@ -8,7 +8,7 @@ pub use super::handler_ee::*; #[cfg(not(feature = "private"))] use { super::KafkaTrigger, - axum::async_trait, + async_trait::async_trait, sqlx::PgConnection, windmill_api_auth::ApiAuthed, windmill_common::{ diff --git a/backend/windmill-trigger-mqtt/src/handler.rs b/backend/windmill-trigger-mqtt/src/handler.rs index 6edcfbabc3..49fb701241 100644 --- a/backend/windmill-trigger-mqtt/src/handler.rs +++ b/backend/windmill-trigger-mqtt/src/handler.rs @@ -1,4 +1,4 @@ -use axum::async_trait; +use async_trait::async_trait; use itertools::Itertools; use sqlx::{types::Json as SqlxJson, PgConnection}; use windmill_api_auth::ApiAuthed; diff --git a/backend/windmill-trigger-nats/src/handler_oss.rs b/backend/windmill-trigger-nats/src/handler_oss.rs index b00d973621..f322335cb8 100644 --- a/backend/windmill-trigger-nats/src/handler_oss.rs +++ b/backend/windmill-trigger-nats/src/handler_oss.rs @@ -8,7 +8,7 @@ use windmill_trigger::TriggerData; #[cfg(not(feature = "private"))] use { super::NatsTrigger, - axum::async_trait, + async_trait::async_trait, sqlx::PgConnection, windmill_api_auth::ApiAuthed, windmill_common::{ diff --git a/backend/windmill-trigger-postgres/src/handler.rs b/backend/windmill-trigger-postgres/src/handler.rs index cbb149af52..dc2f4776fd 100644 --- a/backend/windmill-trigger-postgres/src/handler.rs +++ b/backend/windmill-trigger-postgres/src/handler.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; +use async_trait::async_trait; use axum::{ - async_trait, extract::Path, routing::{delete, get, post}, Extension, Json, Router, @@ -282,10 +282,10 @@ impl TriggerCrud for PostgresTrigger { fn additional_routes(&self) -> Router { Router::new() - .route("/get_template_script/:id", get(get_template_script)) + .route("/get_template_script/{id}", get(get_template_script)) .route("/create_template_script", post(create_template_script)) .route( - "/is_valid_postgres_configuration/*path", + "/is_valid_postgres_configuration/{*path}", get(is_database_in_logical_level), ) .nest("/publication", publication_service()) @@ -296,25 +296,31 @@ impl TriggerCrud for PostgresTrigger { fn publication_service() -> Router { Router::new() - .route("/get/:publication_name/*path", get(get_publication_info)) - .route("/create/:publication_name/*path", post(create_publication)) - .route("/update/:publication_name/*path", post(alter_publication)) + .route("/get/{publication_name}/{*path}", get(get_publication_info)) .route( - "/delete/:publication_name/*path", + "/create/{publication_name}/{*path}", + post(create_publication), + ) + .route( + "/update/{publication_name}/{*path}", + post(alter_publication), + ) + .route( + "/delete/{publication_name}/{*path}", delete(delete_publication), ) - .route("/list/*path", get(list_database_publication)) + .route("/list/{*path}", get(list_database_publication)) } fn slot_service() -> Router { Router::new() - .route("/list/*path", get(list_slot_name)) - .route("/create/*path", post(create_slot)) - .route("/delete/*path", delete(drop_slot_name)) + .route("/list/{*path}", get(list_slot_name)) + .route("/create/{*path}", post(create_slot)) + .route("/delete/{*path}", delete(drop_slot_name)) } fn postgres_service() -> Router { - Router::new().route("/version/*path", get(get_postgres_version)) + Router::new().route("/version/{*path}", get(get_postgres_version)) } async fn check_if_logical_replication_slot_exist( diff --git a/backend/windmill-trigger-sqs/src/handler_oss.rs b/backend/windmill-trigger-sqs/src/handler_oss.rs index 90396e9994..fc72159e24 100644 --- a/backend/windmill-trigger-sqs/src/handler_oss.rs +++ b/backend/windmill-trigger-sqs/src/handler_oss.rs @@ -5,7 +5,7 @@ pub use super::handler_ee::*; #[cfg(not(feature = "private"))] use { super::SqsTrigger, - axum::async_trait, + async_trait::async_trait, sqlx::PgConnection, windmill_api_auth::ApiAuthed, windmill_common::{ diff --git a/backend/windmill-trigger-websocket/src/handler.rs b/backend/windmill-trigger-websocket/src/handler.rs index fd0080950d..223868e96e 100644 --- a/backend/windmill-trigger-websocket/src/handler.rs +++ b/backend/windmill-trigger-websocket/src/handler.rs @@ -1,6 +1,6 @@ use std::borrow::Cow; -use axum::async_trait; +use async_trait::async_trait; use itertools::Itertools; use serde_json::value::RawValue; use sqlx::{types::Json as SqlxJson, PgConnection}; @@ -36,6 +36,7 @@ impl TriggerCrud for WebsocketTrigger { const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ "url", "filters", + "filter_logic", "initial_messages", "url_runnable_args", "can_return_message", @@ -103,6 +104,7 @@ impl TriggerCrud for WebsocketTrigger { is_flow, mode, filters, + filter_logic, initial_messages, url_runnable_args, edited_by, @@ -114,7 +116,7 @@ impl TriggerCrud for WebsocketTrigger { error_handler_args, retry ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), $14, $15, $16 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17 ) "#, w_id, @@ -124,6 +126,7 @@ impl TriggerCrud for WebsocketTrigger { trigger.base.is_flow, trigger.base.mode() as _, &filters as _, + trigger.config.filter_logic, &initial_messages as _, trigger .config @@ -178,26 +181,28 @@ impl TriggerCrud for WebsocketTrigger { path = $3, is_flow = $4, filters = $5, - initial_messages = $6, - url_runnable_args = $7, - edited_by = $8, - permissioned_as = $9, - can_return_message = $10, - can_return_error_result = $11, + filter_logic = $6, + initial_messages = $7, + url_runnable_args = $8, + edited_by = $9, + permissioned_as = $10, + can_return_message = $11, + can_return_error_result = $12, edited_at = now(), server_id = NULL, error = NULL, - error_handler_path = $14, - error_handler_args = $15, - retry = $16 + error_handler_path = $15, + error_handler_args = $16, + retry = $17 WHERE - workspace_id = $12 AND path = $13 + workspace_id = $13 AND path = $14 ", trigger.config.url, trigger.base.script_path, trigger.base.path, trigger.base.is_flow, filters.as_slice() as &[SqlxJson>], + trigger.config.filter_logic, initial_messages.as_slice() as &[SqlxJson>], trigger .config diff --git a/backend/windmill-trigger-websocket/src/lib.rs b/backend/windmill-trigger-websocket/src/lib.rs index ca754d8349..4fe96bb7c2 100644 --- a/backend/windmill-trigger-websocket/src/lib.rs +++ b/backend/windmill-trigger-websocket/src/lib.rs @@ -1,12 +1,9 @@ use std::collections::HashMap; -use windmill_api_auth::ApiAuthed; -use windmill_trigger::trigger_helpers::{ - trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, -}; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use sqlx::{types::Json as SqlxJson, FromRow}; +use windmill_api_auth::ApiAuthed; use windmill_common::{ error::{Error, Result}, jobs::JobTriggerKind, @@ -15,6 +12,9 @@ use windmill_common::{ DB, }; use windmill_queue::PushArgsOwned; +use windmill_trigger::trigger_helpers::{ + trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, +}; pub mod handler; pub mod listener; @@ -30,11 +30,17 @@ impl TriggerJobArgs for WebsocketTrigger { } } +fn default_filter_logic() -> String { + "and".to_string() +} + #[derive(Debug, Clone, FromRow, Serialize, Deserialize)] pub struct WebsocketConfig { pub url: String, #[serde(default)] pub filters: Vec>>, + #[serde(default = "default_filter_logic")] + pub filter_logic: String, #[serde(skip_serializing_if = "Option::is_none")] pub initial_messages: Option>>>, #[serde(skip_serializing_if = "Option::is_none")] @@ -49,6 +55,8 @@ pub struct WebsocketConfig { pub struct WebsocketConfigRequest { url: String, filters: Vec, + #[serde(default = "default_filter_logic")] + filter_logic: String, initial_messages: Option>, url_runnable_args: Option, can_return_message: bool, diff --git a/backend/windmill-trigger-websocket/src/listener.rs b/backend/windmill-trigger-websocket/src/listener.rs index 8aab61d9c6..205f2fe501 100644 --- a/backend/windmill-trigger-websocket/src/listener.rs +++ b/backend/windmill-trigger-websocket/src/listener.rs @@ -18,7 +18,7 @@ use windmill_common::{ DB, }; use windmill_queue::PushArgsOwned; -use windmill_trigger::filter::{is_value_superset, Filter, JsonFilter}; +use windmill_trigger::filter::{check_filters, Filter}; use windmill_trigger::listener::ListeningTrigger; use windmill_trigger::trigger_helpers::{ trigger_runnable, trigger_runnable_and_wait_for_raw_result, @@ -267,26 +267,8 @@ impl Listener for WebsocketTrigger { match msg { tokio_tungstenite::tungstenite::Message::Text(text) => { tracing::debug!("Received text message from WebSocket {}: {}", url, text); - let mut should_handle = true; - for filter in &filters { - match filter { - Filter::JsonFilter(JsonFilter { key, value }) => { - let mut deserializer = serde_json::Deserializer::from_str(text.as_str()); - should_handle = match is_value_superset(&mut deserializer, key, &value) { - Ok(filter_match) => { - filter_match - }, - Err(err) => { - tracing::warn!("Error deserializing filter for WebSocket {}: {:?}", url, err); - false - } - }; - } - } - if !should_handle { - break; - } - } + let use_or = listening_trigger.trigger_config.filter_logic == "or"; + let should_handle = check_filters(&text, &filters, use_or); if should_handle { let trigger_info = HashMap::from([ ("url".to_string(), to_raw_value(&listening_trigger.trigger_config.url)), diff --git a/backend/windmill-trigger/src/filter.rs b/backend/windmill-trigger/src/filter.rs index a1dd59f848..3c9c857058 100644 --- a/backend/windmill-trigger/src/filter.rs +++ b/backend/windmill-trigger/src/filter.rs @@ -80,6 +80,27 @@ where deserializer.deserialize_map(SupersetVisitor { key, value_to_check }) } +pub fn check_filters(text: &str, filters: &[Filter], use_or_logic: bool) -> bool { + if filters.is_empty() { + return true; + } + + let check = |filter: &Filter| -> bool { + match filter { + Filter::JsonFilter(JsonFilter { key, value }) => { + let mut deserializer = serde_json::Deserializer::from_str(text); + is_value_superset(&mut deserializer, key, value).unwrap_or(false) + } + } + }; + + if use_or_logic { + filters.iter().any(check) + } else { + filters.iter().all(check) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index 478f20811d..16f095a903 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -374,11 +374,11 @@ pub fn trigger_routes() -> Router { let mut router = Router::new() .route("/create", post(create_trigger::)) .route("/list", get(list_triggers::)) - .route("/get/*path", get(get_trigger::)) - .route("/update/*path", post(update_trigger::)) - .route("/delete/*path", delete(delete_trigger::)) - .route("/exists/*path", get(exists_trigger::)) - .route("/setmode/*path", post(set_trigger_mode::)); + .route("/get/{*path}", get(get_trigger::)) + .route("/update/{*path}", post(update_trigger::)) + .route("/delete/{*path}", delete(delete_trigger::)) + .route("/exists/{*path}", get(exists_trigger::)) + .route("/setmode/{*path}", post(set_trigger_mode::)); if T::SUPPORTS_TEST_CONNECTION { router = router.route("/test", post(test_connection::)); diff --git a/backend/windmill-types/src/jobs.rs b/backend/windmill-types/src/jobs.rs index 28c3242695..99a5607508 100644 --- a/backend/windmill-types/src/jobs.rs +++ b/backend/windmill-types/src/jobs.rs @@ -118,6 +118,10 @@ impl JobKind { JobKind::FlowDependencies | JobKind::AppDependencies | JobKind::Dependencies ) } + + pub fn is_preview(&self) -> bool { + matches!(self, JobKind::Preview | JobKind::FlowPreview) + } } #[derive(sqlx::FromRow, Debug, Serialize, Clone)] diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index 5eca2be82c..af3a2a9183 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -65,6 +65,7 @@ pub enum ScriptLang { Nu, Java, Ruby, + Rlang, // for related places search: ADD_NEW_LANG } @@ -94,6 +95,7 @@ impl ScriptLang { ScriptLang::Nu => "nu", ScriptLang::Java => "java", ScriptLang::Ruby => "ruby", + ScriptLang::Rlang => "rlang", // for related places search: ADD_NEW_LANG } } @@ -132,7 +134,7 @@ impl ScriptLang { use ScriptLang::*; match self { Nativets | Bun | Bunnative | Deno | Go | Php | CSharp | Java => "//", - Python3 | Bash | Powershell | Graphql | Ansible | Nu | Ruby => "#", + Python3 | Bash | Powershell | Graphql | Ansible | Nu | Ruby | Rlang => "#", Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--", Rust => "//!", // for related places search: ADD_NEW_LANG @@ -167,6 +169,7 @@ impl FromStr for ScriptLang { "nu" => ScriptLang::Nu, "java" => ScriptLang::Java, "ruby" => ScriptLang::Ruby, + "rlang" => ScriptLang::Rlang, // for related places search: ADD_NEW_LANG language => return Err(anyhow::anyhow!("{} is currently not supported", language)), }; @@ -450,6 +453,8 @@ pub struct ScriptHistory { pub script_hash: ScriptHash, #[serde(skip_serializing_if = "Option::is_none")] pub deployment_msg: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub created_at: Option>, } #[derive(Deserialize)] @@ -510,6 +515,8 @@ pub struct NewScript { pub assets: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub modules: Option>, + #[serde(default)] + pub auto_parent: Option, } // IMPORTANT: update this Hash impl when adding fields to NewScript diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index c1e2a4927a..4e2357b14e 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-worker-volumes/private", "windmill-queue/private"] +private = ["windmill-worker-volumes/private", "windmill-queue/private", "windmill-common/private"] mcp = ["dep:windmill-mcp"] prometheus = ["dep:prometheus", "windmill-common/prometheus"] enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker-volumes/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] @@ -36,6 +36,7 @@ rust = ["dep:windmill-parser-rust"] nu = ["dep:windmill-parser-nu"] java = ["dep:windmill-parser-java"] ruby = ["dep:windmill-parser-ruby"] +rlang = ["dep:windmill-parser-r"] duckdb = ["dep:libloading"] quickjs = ["windmill-jseval/quickjs"] bedrock = ["dep:aws-sdk-bedrockruntime", "windmill-common/bedrock", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-types"] @@ -60,6 +61,7 @@ windmill-parser-csharp = { workspace = true, optional = true } windmill-parser-nu = { workspace = true, optional = true } windmill-parser-java = { workspace = true, optional = true } windmill-parser-ruby = { workspace = true, optional = true } +windmill-parser-r = { workspace = true, optional = true } windmill-parser-py = { workspace = true, optional = true } windmill-parser-yaml.workspace = true windmill-parser-py-imports = { workspace = true, optional = true } @@ -146,6 +148,7 @@ rcgen = { workspace = true, optional = true } [dev-dependencies] tempfile.workspace = true +x509-parser.workspace = true [build-dependencies] libffi-sys = { workspace = true, optional = true } diff --git a/backend/windmill-worker/nsjail/install.r.config.proto b/backend/windmill-worker/nsjail/install.r.config.proto new file mode 100644 index 0000000000..8163be8bbb --- /dev/null +++ b/backend/windmill-worker/nsjail/install.r.config.proto @@ -0,0 +1,100 @@ +name: "r install" + +mode: ONCE +hostname: "r" +log_level: ERROR +time_limit: 900 + +disable_rl: true + +envar: "HOME=/tmp" +envar: "R_INSTALL_TAR=/usr/bin/tar --no-same-owner" + +cwd: "/tmp" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: true +keep_env: true +mount_proc: true + + +mount { + src: "/bin" + dst: "/bin" + is_bind: true + mandatory: false +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true + mandatory: false +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true + mandatory: false +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + src: "{JOB_DIR}" + dst: "/tmp" + is_bind: true + mandatory: false + rw: true +} + +mount { + src: "{PKG_DIR}" + dst: "/install" + is_bind: true + rw: true +} + +mount { + src: "/sys/devices/system/cpu" + dst: "/sys/devices/system/cpu" + is_bind: true + mandatory: false +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +mount { + src: "{TRACING_PROXY_CA_CERT_PATH}" + dst: "{TRACING_PROXY_CA_CERT_PATH}" + is_bind: true + mandatory: false +} + +#{DEV} diff --git a/backend/windmill-worker/nsjail/run.r.config.proto b/backend/windmill-worker/nsjail/run.r.config.proto new file mode 100644 index 0000000000..72c30f489b --- /dev/null +++ b/backend/windmill-worker/nsjail/run.r.config.proto @@ -0,0 +1,125 @@ +name: "r run script" + +mode: ONCE +hostname: "r" +log_level: ERROR + +disable_rl: true + +cwd: "/tmp" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: false +keep_env: true +# mount_proc: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true + mandatory: false +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true + mandatory: false +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true + mandatory: false +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + dst: "/tmp" + fstype: "tmpfs" + rw: true + options: "size=500000000" +} + + +mount { + src: "{JOB_DIR}/main.r" + dst: "/tmp/main.r" + is_bind: true + mandatory: false +} + +mount { + src: "{JOB_DIR}/args.json" + dst: "/tmp/args.json" + is_bind: true +} + +mount { + src: "{JOB_DIR}/result.json" + dst: "/tmp/result.json" + rw: true + is_bind: true +} + +mount { + src: "{R_CACHE_DIR}" + dst: "{R_CACHE_DIR}" + is_bind: true + mandatory: false +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +mount { + src: "/sys/devices/system/cpu" + dst: "/sys/devices/system/cpu" + is_bind: true + mandatory: false +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +iface_no_lo: true + +{SHARED_MOUNT} + +mount { + src: "{TRACING_PROXY_CA_CERT_PATH}" + dst: "{TRACING_PROXY_CA_CERT_PATH}" + is_bind: true + mandatory: false +} + +#{DEV} diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 6bc6cacc32..433f96bcec 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1092,7 +1092,12 @@ pub async fn prebundle_bun_script( } let origin = format!("{job_dir}/main.js"); - write_file(job_dir, "main.ts", &remove_pinned_imports(inner_content)?)?; + let mut content = remove_pinned_imports(inner_content)?; + if crate::wac_executor::is_wac_v2_ts(inner_content) { + content = crate::wac_executor::inject_wac_task_names(&content); + content = format!("export {{ WorkflowCtx, StepSuspend, setWorkflowCtx }} from \"windmill-client\";\n{content}"); + } + write_file(job_dir, "main.ts", &content)?; build_loader( job_dir, base_internal_url, @@ -1318,29 +1323,12 @@ pub async fn handle_bun_job( // Also handles: export const, let, var, and optional generic type parameters. // Skips calls that already have a string argument: `task("path", async ...` let inner_content = if is_wac_v2 { - use regex::Regex; - use std::borrow::Cow; - lazy_static::lazy_static! { - static ref TASK_RE: Regex = - Regex::new(r#"(?m)((?:export\s+)?(?:const|let|var)\s+)(\w+)(\s*=\s*task\s*(?:<[^>]*>)?\s*\(\s*)(async\b)"#).unwrap(); - } - let replaced = TASK_RE.replace_all(inner_content, r#"${1}${2}${3}"${2}", ${4}"#); - match replaced { - Cow::Borrowed(_) => inner_content.to_string(), - Cow::Owned(s) => s, - } + crate::wac_executor::inject_wac_task_names(inner_content) } else { inner_content.to_string() }; let inner_content = inner_content.as_str(); - // WAC v2 scripts can't use bundle caching because the wrapper imports - // windmill-client from node_modules, which isn't available in bundle mode - if is_wac_v2 && has_bundle_cache { - has_bundle_cache = false; - let _ = write_file(job_dir, "main.ts", inner_content)?; - } - let mut format = BundleFormat::Cjs; if has_bundle_cache { let target; @@ -1561,6 +1549,12 @@ pub async fn handle_bun_job( "./main.ts" }; + let wac_client_import = if has_bundle_cache { + "./main.js" + } else { + "windmill-client" + }; + let preprocessor = if let Some(pre_args) = pre_args { let pre_spread = pre_args.into_iter().map(|x| x.name).join(","); format!( @@ -1588,7 +1582,7 @@ pub async fn handle_bun_job( format!( r#" import * as Main from "{main_import}"; -import {{ WorkflowCtx, StepSuspend, setWorkflowCtx }} from "windmill-client"; +import {{ WorkflowCtx, StepSuspend, setWorkflowCtx }} from "{wac_client_import}"; import * as fs from "fs/promises"; @@ -1779,7 +1773,6 @@ try {{ && !annotation.nobundling && !*DISABLE_BUNDLING && !codebase.is_some() - && !is_wac_v2 && (maybe_lock.get_lock().is_some() || annotation.native); let write_loader_f = async { @@ -1844,6 +1837,17 @@ try {{ } } + // Prepend WAC re-exports to main.ts so the bundle includes WorkflowCtx etc. + if build_cache && is_wac_v2 { + let main_path = format!("{job_dir}/main.ts"); + let current = read_file_content(&main_path).await?; + write_file( + job_dir, + "main.ts", + &format!("export {{ WorkflowCtx, StepSuspend, setWorkflowCtx }} from \"windmill-client\";\n{current}"), + )?; + } + if !codebase.is_some() && !has_bundle_cache { if build_cache { generate_bun_bundle( @@ -1882,14 +1886,14 @@ try {{ } if !annotation.native { let ex_wrapper = read_file_content(&format!("{job_dir}/wrapper.mjs")).await?; - write_file( - job_dir, - "wrapper.mjs", - &ex_wrapper.replace( - "import * as Main from \"./main.ts\"", - "import * as Main from \"./main.js\"", - ), - )?; + let mut rewritten = ex_wrapper.replace( + "import * as Main from \"./main.ts\"", + "import * as Main from \"./main.js\"", + ); + if is_wac_v2 { + rewritten = rewritten.replace("from \"windmill-client\"", "from \"./main.js\""); + } + write_file(job_dir, "wrapper.mjs", &rewritten)?; write_file(job_dir, "package.json", r#"{ "type": "module" }"#)?; } fs::remove_file(format!("{job_dir}/main.ts"))?; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index c6d737becb..d303771cc8 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -145,7 +145,7 @@ pub async fn write_file_binary(dir: &str, path: &str, content: &[u8]) -> error:: } lazy_static::lazy_static! { - static ref RE_RES_VAR: Regex = Regex::new(r#"\$(?:var|res|encrypted)\:"#).unwrap(); + static ref RE_RES_VAR: Regex = Regex::new(r#"\$(?:var|jsonvar|res|encrypted)\:"#).unwrap(); } pub async fn transform_json<'a>( @@ -255,6 +255,15 @@ pub async fn transform_json_value( Error::NotFound(format!("Variable {path} not found for `{name}`: {e:#}")) }) } + Value::String(y) if y.starts_with("$jsonvar:") => { + let path = y.strip_prefix("$jsonvar:").unwrap(); + let v = client.get_variable_value(path).await.map_err(|e| { + Error::NotFound(format!("Variable {path} not found for `{name}`: {e:#}")) + })?; + serde_json::from_str::(&v).map_err(|e| { + Error::internal_err(format!("Failed to parse $jsonvar value as JSON: {e}")) + }) + } Value::String(y) if y.starts_with("$res:") => { let path = y.strip_prefix("$res:").unwrap(); diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index e97ec8ff9a..bb32373b76 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -17,6 +17,9 @@ mod java_executor; #[cfg(feature = "ruby")] mod ruby_executor; +#[cfg(feature = "rlang")] +mod r_executor; + mod ai; mod ai_executor; mod bun_executor; diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 5b309a611c..f769e1034d 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -285,7 +285,16 @@ pub async fn do_postgresql( annotations.result_collection }; - let database_string = database.to_uri(); + let use_iam_auth = database.use_iam_auth == Some(true); + + // Include use_iam_auth in cache key to distinguish IAM vs non-IAM connections to the same host. + // The cache key is static (doesn't include the token), which is correct because PostgreSQL + // connections remain valid after initial auth — fresh tokens are generated on cache miss. + let database_string = if use_iam_auth { + format!("{}?iam=true", database.to_uri()) + } else { + database.to_uri() + }; let database_string_clone = database_string.clone(); let mtex; @@ -309,7 +318,20 @@ pub async fn do_postgresql( ); (None, mtex) } else { - let (client, connection) = database.connect().await?; + let (client, connection) = if use_iam_auth { + #[cfg(all(feature = "enterprise", feature = "private"))] + { + database.connect_with_iam().await? + } + #[cfg(not(all(feature = "enterprise", feature = "private")))] + { + return Err(Error::ExecutionErr( + "IAM RDS authentication requires Windmill Enterprise Edition".to_string(), + )); + } + } else { + database.connect().await? + }; let handle = tokio::spawn(async move { if let Err(e) = connection.await { let mut mtex = CONNECTION_CACHE.lock().await; diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index 89d3bd5b62..d3d0fb36fb 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -84,8 +84,27 @@ pub async fn composer_install( ) -> Result { check_executor_binary_exists("php", PHP_PATH.as_str(), "php")?; - // When a lock file is available the dependency set is fully pinned, so we - // can cache the installed vendor/ directory and reuse it across executions. + // When no lock is provided (previews), try to reuse a previously resolved + // lockfile from the DB so we can hit the same vendor cache as deployed scripts. + let lock = if lock.is_none() && !*COMPOSER_VENDOR_CACHE_DISABLED { + let req_hash = format!("composer-{}", calculate_hash(&requirements)); + if let Some(db) = conn.as_sql() { + sqlx::query_scalar!( + "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", + req_hash + ) + .fetch_optional(db) + .await + .ok() + .flatten() + } else { + None + } + } else { + lock + }; + + // Cache the installed vendor/ directory keyed by requirements + lock content. // Set COMPOSER_VENDOR_CACHE_DISABLED=1 to opt out. let vendor_cache_hit = if !*COMPOSER_VENDOR_CACHE_DISABLED { if let Some(ref lock_content) = lock { @@ -171,6 +190,9 @@ pub async fn composer_install( ) .await?; + // lock was `None` means composer resolved deps from scratch (no lock from + // caller or DB). This is the only case where we should update the DB cache. + let freshly_resolved = lock.is_none(); let resolved_lock = match lock { Some(l) => l, None => { @@ -195,6 +217,27 @@ pub async fn composer_install( { tracing::warn!("Could not save composer vendor dir to cache: {e:?}"); } + + // Cache the resolved lockfile in the DB so future previews (which lack a + // lock file) can look it up by requirements hash and hit the same vendor + // cache. TTL of 7 days keeps previews reasonably fresh. + // Only write when composer resolved from scratch (no lock from caller or + // DB) to avoid endlessly refreshing the TTL on stale resolutions. + if freshly_resolved { + let req_hash = format!("composer-{}", calculate_hash(&requirements)); + if let Some(db) = conn.as_sql() { + if let Err(e) = sqlx::query!( + "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('7 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = EXCLUDED.lockfile, expiration = EXCLUDED.expiration", + req_hash, + &resolved_lock + ) + .execute(db) + .await + { + tracing::warn!("Could not cache composer lockfile resolution: {e:?}"); + } + } + } } Ok(format!( diff --git a/backend/windmill-worker/src/r_executor.rs b/backend/windmill-worker/src/r_executor.rs new file mode 100644 index 0000000000..fa8a1809b1 --- /dev/null +++ b/backend/windmill-worker/src/r_executor.rs @@ -0,0 +1,715 @@ +use std::{collections::HashMap, process::Stdio}; + +use itertools::Itertools; +use tokio::{ + fs::{self, File}, + io::{AsyncReadExt, AsyncWriteExt}, + process::Command, +}; +use uuid::Uuid; +use windmill_common::{ + client::AuthedClient, + error::Error, + utils::calculate_hash, + worker::{write_file, Connection, RlangAnnotations}, +}; +use windmill_parser::Arg; +use windmill_parser_r::{parse_r_requirements, parse_r_signature}; +use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; + +use crate::{ + common::{ + build_command_with_isolation, create_args_and_out_file, get_reserved_variables, + read_result, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, + }, + get_proxy_envs_for_lang, + handle_child::{self}, + is_sandboxing_enabled, + universal_pkg_installer::{ + par_install_language_dependencies_seq, DependencyGraph, InstallDeps, RequiredDependency, + }, + DISABLE_NUSER, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, R_CACHE_DIR, + TRACING_PROXY_CA_CERT_PATH, +}; +use windmill_common::scripts::ScriptLang; + +lazy_static::lazy_static! { + static ref RSCRIPT_PATH: String = std::env::var("RSCRIPT_PATH").unwrap_or_else(|_| "/usr/bin/Rscript".to_string()); + static ref R_CONCURRENT_DOWNLOADS: usize = std::env::var("R_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(5)).unwrap_or(5); + static ref R_PROXY_ENVS: Vec<(String, String)> = { + PROXY_ENVS + .clone() + .into_iter() + .map(|(k, v)| (k.to_lowercase(), v)) + .collect() + }; +} + +const NSJAIL_CONFIG_RUN_R_CONTENT: &str = include_str!("../nsjail/run.r.config.proto"); +const NSJAIL_CONFIG_INSTALL_R_CONTENT: &str = include_str!("../nsjail/install.r.config.proto"); + +#[allow(dead_code)] +pub(crate) struct JobHandlerInput<'a> { + pub base_internal_url: &'a str, + pub canceled_by: &'a mut Option, + pub client: &'a AuthedClient, + pub parent_runnable_path: Option, + pub conn: &'a Connection, + pub envs: HashMap, + pub inner_content: &'a str, + pub job: &'a MiniPulledJob, + pub job_dir: &'a str, + pub mem_peak: &'a mut i32, + pub occupancy_metrics: &'a mut OccupancyMetrics, + pub requirements_o: Option<&'a String>, + pub shared_mount: &'a str, + pub worker_name: &'a str, +} + +pub async fn handle_r_job<'a>( + mut args: JobHandlerInput<'a>, +) -> Result, Error> { + let annotation = RlangAnnotations::parse(args.inner_content); + + if !std::path::Path::new(RSCRIPT_PATH.as_str()).exists() { + return Err(Error::ExecutionErr(format!( + "Rscript binary not found at '{}'. R is only available in the windmill-full (CE) or windmill-ee-full (EE) Docker images.", + *RSCRIPT_PATH + ))); + } + + if annotation.sandbox && NSJAIL_AVAILABLE.is_none() { + return Err(Error::ExecutionErr( + "Script has #sandbox annotation but nsjail is not available on this worker. \ + Please ensure nsjail is installed or remove the #sandbox annotation." + .to_string(), + )); + } + + // --- Prepare --- + { + prepare(&args).await?; + } + // --- Resolve lockfile --- + let lockfile = resolve( + &args.job.id, + args.inner_content, + args.mem_peak, + args.canceled_by, + args.job_dir, + args.conn, + args.worker_name, + &args.job.workspace_id, + annotation.renv_verbose, + ) + .await?; + // --- Install --- + let lib_path = if !lockfile.is_empty() { + Some( + install( + &mut args, + &lockfile, + annotation.renv_verbose, + annotation.renv_install_verbose, + ) + .await?, + ) + } else { + None + }; + // --- Execute --- + { + run(&mut args, lib_path.as_deref(), annotation.sandbox).await?; + } + // --- Retrieve results --- + { + read_result(&args.job_dir, None).await + } +} + +pub async fn prepare<'a>( + JobHandlerInput { job, conn, job_dir, inner_content, client, .. }: &JobHandlerInput<'a>, +) -> Result<(), Error> { + create_args_and_out_file(&client, job, job_dir, conn).await?; + File::create(format!("{}/main.r", job_dir)) + .await? + .write_all(&wrap(inner_content)?.into_bytes()) + .await?; + + // Create windmill client library for R + let wm_lib_path = format!("{}/r_libs", *R_CACHE_DIR); + fs::create_dir_all(&wm_lib_path).await?; + { + File::create(format!("{}/windmill.r", &wm_lib_path)) + .await? + .write_all( + r##" +# Windmill mini client methods for R +# Uses base R url() + readLines() to avoid requiring any extra R packages + +.wm_fetch_raw <- function(url) { + token <- Sys.getenv("WM_TOKEN") + con <- url(url, headers = c(Authorization = paste("Bearer", token))) + on.exit(close(con)) + paste(readLines(con, warn = FALSE), collapse = "\n") +} + +get_variable <- function(path) { + base_url <- Sys.getenv("BASE_INTERNAL_URL") + workspace <- Sys.getenv("WM_WORKSPACE") + url <- paste0(base_url, "/api/w/", workspace, "/variables/get_value/", path) + jsonlite::fromJSON(.wm_fetch_raw(url)) +} + +get_resource <- function(path) { + base_url <- Sys.getenv("BASE_INTERNAL_URL") + workspace <- Sys.getenv("WM_WORKSPACE") + url <- paste0(base_url, "/api/w/", workspace, "/resources/get_value_interpolated/", path) + jsonlite::fromJSON(.wm_fetch_raw(url)) +} +"## + .as_bytes(), + ) + .await?; + } + Ok(()) +} + +pub async fn resolve<'a>( + job_id: &Uuid, + inner_content: &str, + mem_peak: &mut i32, + canceled_by: &mut Option, + job_dir: &str, + conn: &Connection, + worker_name: &str, + w_id: &str, + verbose: bool, +) -> Result { + let mut packages = parse_r_requirements(inner_content)?; + + // jsonlite is always needed by the wrapper for JSON arg parsing and result serialization + let has_jsonlite = packages.lines().any(|l| l.trim() == "jsonlite"); + if !has_jsonlite { + if packages.is_empty() { + packages = "jsonlite".to_string(); + } else { + packages.push_str("\njsonlite"); + } + } + + // Check cache + let req_hash = format!("r-{}", calculate_hash(&packages)); + if let Some(db) = conn.as_sql() { + if let Some(cached) = sqlx::query_scalar!( + "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", + req_hash + ) + .fetch_optional(db) + .await? + { + return Ok(cached); + } + } + + append_logs( + job_id, + w_id, + format!("\n--- RESOLVING R PACKAGES ---\n"), + conn, + ) + .await; + + // main.r is already written by prepare() and contains the library() calls. + // renv will scan it to detect dependencies. + // Disable renv's own package cache — Windmill manages its own install cache. + let resolve_script = format!( + r#"options( + repos = c(CRAN = "https://cloud.r-project.org"), + renv.verbose = {verbose_r}, + renv.config.cache.enabled = FALSE, + renv.config.restart.enabled = FALSE, + renv.config.synchronized.check = FALSE +) +renv::consent(provided = TRUE) +suppressMessages(renv::init(bare = TRUE, restart = FALSE)) +suppressMessages(renv::install(prompt = FALSE)) +suppressMessages(renv::snapshot(type = "implicit", prompt = FALSE)) +"#, + verbose_r = if verbose { "TRUE" } else { "FALSE" }, + ); + + let mut file = File::create(format!("{}/resolve.r", job_dir)).await?; + file.write_all(resolve_script.as_bytes()).await?; + + let child = { + let renv_root = format!("{}/renv", *R_CACHE_DIR); + let rscript_executable = if cfg!(windows) { + "Rscript.exe" + } else { + RSCRIPT_PATH.as_str() + }; + let mut cmd = Command::new(rscript_executable); + cmd.current_dir(job_dir) + .env("PATH", PATH_ENV.as_str()) + .env("RENV_PATHS_ROOT", &renv_root) + .arg("resolve.r") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + start_child_process(cmd, rscript_executable, false).await? + }; + handle_child::handle_child( + job_id, + conn, + mem_peak, + canceled_by, + child, + false, + worker_name, + w_id, + "r resolve", + None, + false, + &mut None, + None, + None, + ) + .await?; + + let lock_path = format!("{}/renv.lock", job_dir); + let mut lock_file = File::open(&lock_path).await?; + let mut lock = String::new(); + lock_file.read_to_string(&mut lock).await?; + + // Cache the lockfile + if let Some(db) = conn.as_sql() { + sqlx::query!( + "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = EXCLUDED.lockfile", + req_hash, + lock.clone(), + ).fetch_optional(db).await?; + } + + // Log a compact summary instead of the entire renv.lock JSON + let pkg_count = serde_json::from_str::(&lock) + .ok() + .and_then(|v| v.get("Packages")?.as_object().map(|o| o.len())) + .unwrap_or(0); + append_logs( + job_id, + w_id, + format!("resolved {} packages\n", pkg_count), + conn, + ) + .await; + Ok(lock) +} + +struct RenvPackage { + name: String, + version: String, + repo_url: String, + /// Package names from Imports + Depends fields + dependencies: Vec, +} + +/// Parse renv.lock JSON and extract package info including dependency edges. +fn parse_renv_lock(lockfile: &str) -> Result, Error> { + let lock: serde_json::Value = serde_json::from_str(lockfile) + .map_err(|e| Error::ExecutionErr(format!("Failed to parse renv.lock: {}", e)))?; + + // Build repo name -> URL map from R.Repositories + let mut repo_urls: HashMap = HashMap::new(); + if let Some(repos) = lock + .get("R") + .and_then(|r| r.get("Repositories")) + .and_then(|r| r.as_array()) + { + for repo in repos { + if let (Some(name), Some(url)) = ( + repo.get("Name").and_then(|v| v.as_str()), + repo.get("URL").and_then(|v| v.as_str()), + ) { + repo_urls.insert(name.to_string(), url.to_string()); + } + } + } + + let packages = lock + .get("Packages") + .and_then(|p| p.as_object()) + .ok_or_else(|| Error::ExecutionErr("renv.lock missing Packages field".to_string()))?; + + let mut result = vec![]; + for (_name, pkg) in packages { + let pkg_name = pkg + .get("Package") + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string(); + let version = pkg + .get("Version") + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string(); + let repo_name = pkg + .get("Repository") + .and_then(|v| v.as_str()) + .unwrap_or("CRAN"); + let repo_url = repo_urls + .get(repo_name) + .cloned() + .unwrap_or_else(|| "https://cloud.r-project.org".to_string()); + + let mut dependencies = vec![]; + if let Some(imports) = pkg.get("Imports").and_then(|v| v.as_array()) { + for entry in imports { + if let Some(s) = entry.as_str() { + // Entries look like "cli (>= 3.6.2)" — take just the name + let name = s.split_whitespace().next().unwrap_or(""); + if !name.is_empty() && name != "R" { + dependencies.push(name.to_string()); + } + } + } + } + + // Skip renv itself — it's already loaded and reinstalling it while + // loaded triggers a noisy "Restart your R session" message. + if !pkg_name.is_empty() && !version.is_empty() && pkg_name != "renv" { + result.push(RenvPackage { name: pkg_name, version, repo_url, dependencies }); + } + } + Ok(result) +} + +async fn install<'a>( + args: &mut JobHandlerInput<'a>, + lockfile: &str, + verbose: bool, + install_verbose: bool, +) -> Result { + let lib_path = format!("{}/r_site_library", *R_CACHE_DIR); + fs::create_dir_all(&lib_path).await?; + + let packages = parse_renv_lock(lockfile)?; + if packages.is_empty() { + return Ok(lib_path); + } + + #[derive(Clone, Debug)] + struct RPackagePayload { + pkg: String, + version: String, + #[allow(dead_code)] + repo_url: String, + } + + // Build dependency graph for topological layering + let mut graph = DependencyGraph::new(); + for renv_pkg in &packages { + let handle = format!("{}-{}", renv_pkg.name, renv_pkg.version); + // renv uses staged installation: it builds to a temp dir then rename()s onto + // the target. If the target is a bind mount point, rename fails with + // "target file already exists". We work around this by mounting the parent + // (wrapper) dir at /install so renv can freely create /install/{pkg}/ via rename. + let pkg_outer = format!("{}/{}_outer", lib_path, renv_pkg.name); + let path = format!("{}/{}", pkg_outer, renv_pkg.name); + graph.insert( + renv_pkg.name.clone(), + RequiredDependency { + path, + _s3_handle: handle, + display_name: format!("{} ({})", renv_pkg.name, renv_pkg.version), + custom_payload: RPackagePayload { + pkg: renv_pkg.name.clone(), + version: renv_pkg.version.clone(), + repo_url: renv_pkg.repo_url.clone(), + }, + }, + renv_pkg.dependencies.clone(), + ); + } + + let jailed = !cfg!(windows) && is_sandboxing_enabled(); + let job_dir = args.job_dir.to_owned(); + + par_install_language_dependencies_seq( + InstallDeps::Layered(graph), + "r", + "Rscript", + false, + *R_CONCURRENT_DOWNLOADS, + move |dependency| { + let lib_path_c = lib_path.clone(); + let job_dir = job_dir.clone(); + let pkg_name = &dependency.custom_payload.pkg; + // pkg_outer is the wrapper dir mounted rw at /install inside nsjail. + // renv creates /install/{pkg}/ inside it via staged rename. + let pkg_outer = format!("{}/{}_outer", lib_path_c, pkg_name); + std::fs::create_dir_all(&pkg_outer)?; + + let mut cmd = if jailed { + let nsjail_proto = format!("{}.install.config.proto", Uuid::new_v4()); + let config_content = NSJAIL_CONFIG_INSTALL_R_CONTENT + .replace("{JOB_DIR}", &job_dir) + .replace("{PKG_DIR}", &pkg_outer) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("#{DEV}", DEV_CONF_NSJAIL); + let _ = write_file( + &job_dir, + &nsjail_proto, + &config_content, + )?; + let mut cmd = Command::new(NSJAIL_PATH.as_str()); + cmd.args(vec![ + "--config", + &nsjail_proto, + "--", + RSCRIPT_PATH.as_str(), + ]); + cmd + } else { + Command::new(if cfg!(windows) { + "Rscript.exe" + } else { + RSCRIPT_PATH.as_str() + }) + }; + + let verbose_r = if verbose { "TRUE" } else { "FALSE" }; + let install_verbose_r = if install_verbose { "TRUE" } else { "FALSE" }; + let install_lib = if jailed { "/install".to_string() } else { pkg_outer.clone() }; + cmd.env_clear() + .current_dir(&job_dir) + .env("PATH", PATH_ENV.as_str()) + .envs(R_PROXY_ENVS.clone()); + cmd + .args(&[ + "-e", + &format!( + r#"options(renv.verbose = {verbose_r}, renv.config.install.verbose = {install_verbose_r}, renv.config.restart.enabled = FALSE); renv::install("{pkg}@{version}", library = "{lib}", dependencies = FALSE)"#, + verbose_r = verbose_r, + install_verbose_r = install_verbose_r, + pkg = dependency.custom_payload.pkg, + version = dependency.custom_payload.version, + lib = install_lib, + ), + // install.packages fallback (no version pinning): + // &format!( + // r#"install.packages("{pkg}", lib = "{lib}", repos = "{repo}", dependencies = FALSE, quiet = {quiet}, INSTALL_opts = "--no-test-load --no-lock")"#, + // pkg = dependency.custom_payload.pkg, + // lib = install_lib, + // repo = dependency.custom_payload.repo_url, + // quiet = quiet_flag, + // ), + ]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + Ok(cmd) + }, + None, + &args.job.id, + &args.job.workspace_id, + args.worker_name, + jailed, + args.conn, + ) + .await?; + + Ok(format!("{}/r_site_library", *R_CACHE_DIR)) +} + +/// Build R_LIBS_USER from lib_path by listing *_outer subdirs. +/// Each package wrapper dir ({pkg}_outer) is added so R finds {pkg}_outer/{pkg}/DESCRIPTION. +fn r_libs_user(lib_path: &str) -> String { + std::fs::read_dir(lib_path) + .into_iter() + .flatten() + .filter_map(|e| e.ok()) + .filter(|e| { + e.file_type().map(|t| t.is_dir()).unwrap_or(false) + && e.file_name().to_string_lossy().ends_with("_outer") + }) + .map(|e| e.path().to_string_lossy().to_string()) + .collect::>() + .join(":") +} + +async fn run<'a>( + JobHandlerInput { + occupancy_metrics, + mem_peak, + canceled_by, + worker_name, + job, + conn, + job_dir, + shared_mount, + client, + envs, + base_internal_url, + parent_runnable_path, + .. + }: &mut JobHandlerInput<'a>, + lib_path: Option<&str>, + sandbox: bool, +) -> Result<(), Error> { + let reserved_variables = + get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?; + + let nsjail = !cfg!(windows) && (is_sandboxing_enabled() || sandbox); + let child = if nsjail { + append_logs( + &job.id, + &job.workspace_id, + "\n--- R CODE EXECUTION (nsjail) ---\n".to_string(), + conn, + ) + .await; + + write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_R_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{SHARED_MOUNT}", &shared_mount) + .replace("{R_CACHE_DIR}", &*R_CACHE_DIR) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), + )?; + let mut cmd = Command::new(NSJAIL_PATH.as_str()); + cmd.env_clear() + .current_dir(job_dir) + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .envs(envs) + .envs(reserved_variables) + .envs(R_PROXY_ENVS.clone()) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Rlang, &job.id, &job.workspace_id, conn) + .await?, + ); + if let Some(lp) = lib_path { + cmd.env("R_LIBS_USER", r_libs_user(lp)); + } + cmd.args(vec![ + "--config", + "run.config.proto", + "--", + RSCRIPT_PATH.as_str(), + "main.r", + ]); + cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); + + start_child_process(cmd, NSJAIL_PATH.as_str(), false).await? + } else { + append_logs( + &job.id, + &job.workspace_id, + format!("\n--- R CODE EXECUTION ---\n"), + conn, + ) + .await; + + let rscript_executable = if cfg!(windows) { + "Rscript.exe" + } else { + RSCRIPT_PATH.as_str() + }; + + let args = vec!["main.r"]; + let mut cmd = build_command_with_isolation(rscript_executable, &args); + + cmd.env_clear() + .current_dir(job_dir.to_owned()) + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .envs(reserved_variables) + .envs(R_PROXY_ENVS.clone()) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Rlang, &job.id, &job.workspace_id, conn) + .await?, + ) + .envs(envs); + if let Some(lp) = lib_path { + cmd.env("R_LIBS_USER", r_libs_user(lp)); + } + + cmd.stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str()) + .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ); + } + start_child_process(cmd, rscript_executable, false).await? + }; + handle_child::handle_child( + &job.id, + conn, + mem_peak, + canceled_by, + child, + nsjail, + worker_name, + &job.workspace_id, + "r", + job.timeout, + false, + &mut Some(occupancy_metrics), + None, + None, + ) + .await?; + Ok(()) +} + +fn wrap(inner_content: &str) -> Result { + let sig = parse_r_signature(inner_content)?; + let spread = sig + .args + .clone() + .into_iter() + .map(|Arg { name, .. }| format!("{name} = args${name}", name = name)) + .collect_vec() + .join(", "); + let wm_lib_path = format!("{}/r_libs/windmill.r", *R_CACHE_DIR); + Ok(format!( + r#"source("{wm_lib_path}") + +suppressPackageStartupMessages({{ +{inner_content} +}}) + +library(jsonlite) +args <- fromJSON("args.json") + +tryCatch({{ + res <- main({spread}) + write(toJSON(res, auto_unbox = TRUE, null = "null"), "result.json") +}}, error = function(e) {{ + error_obj <- list( + name = class(e)[1], + message = conditionMessage(e), + stack = paste(capture.output(traceback()), collapse = "\n") + ) + write(toJSON(error_obj, auto_unbox = TRUE), "result.json") + stop(e) +}}) +"#, + wm_lib_path = wm_lib_path, + inner_content = inner_content, + spread = spread, + )) +} diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index 82ed16eab0..d94864766b 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -29,7 +29,7 @@ use crate::{ get_proxy_envs_for_lang, handle_child::{self}, is_sandboxing_enabled, read_ee_registry_url_list_with_workspace_override, - universal_pkg_installer::{par_install_language_dependencies_seq, RequiredDependency}, + universal_pkg_installer::{par_install_language_dependencies_seq, InstallDeps, RequiredDependency}, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUBY_CACHE_DIR, RUBY_REPOS, TRACING_PROXY_CA_CERT_PATH, }; @@ -618,7 +618,7 @@ async fn install<'a>( get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?, ); par_install_language_dependencies_seq( - deps.clone(), + InstallDeps::Flat(deps.clone()), "ruby", "gem", false, @@ -721,7 +721,7 @@ async fn install<'a>( Ok(cmd) }, - // async move |_| Ok(()), + None, &job.id, &job.workspace_id, worker_name, diff --git a/backend/windmill-worker/src/universal_pkg_installer.rs b/backend/windmill-worker/src/universal_pkg_installer.rs index f129371ca1..47b8401184 100644 --- a/backend/windmill-worker/src/universal_pkg_installer.rs +++ b/backend/windmill-worker/src/universal_pkg_installer.rs @@ -1,3 +1,4 @@ +use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::Arc; use anyhow::bail; @@ -31,6 +32,153 @@ pub struct RequiredDependency { pub custom_payload: T, } +/// Generic dependency graph that produces topologically sorted layers via Kahn's algorithm. +/// Each layer's packages only depend on packages from earlier layers, enabling parallel install +/// per layer. +#[allow(dead_code)] +pub struct DependencyGraph { + nodes: HashMap>, + deps: HashMap>, +} + +#[allow(dead_code)] +impl DependencyGraph { + pub fn new() -> Self { + Self { nodes: HashMap::new(), deps: HashMap::new() } + } + + /// Insert a dependency and the names of packages it depends on. + /// References to packages not in the graph are silently ignored during layering. + pub fn insert( + &mut self, + key: impl Into, + dep: RequiredDependency, + depends_on: Vec, + ) { + let key = key.into(); + self.nodes.insert(key.clone(), dep); + self.deps.insert(key, depends_on.into_iter().collect()); + } + + /// Render a dependency tree string. Each package appears once, nested under the first parent + /// that pulls it in. Only includes packages present in `filter` (if provided). + pub fn print_tree(&self, filter: Option<&HashSet>) -> String { + // Find roots: packages nothing else in the graph depends on + let mut depended_on: HashSet<&str> = HashSet::new(); + for dep_set in self.deps.values() { + for to in dep_set { + if self.nodes.contains_key(to) { + depended_on.insert(to.as_str()); + } + } + } + let roots: Vec<&String> = self + .nodes + .keys() + .filter(|k| !depended_on.contains(k.as_str())) + .filter(|k| filter.map_or(true, |f| f.contains(*k))) + .sorted() + .collect(); + + let mut out = String::new(); + let mut seen = HashSet::new(); + for root in roots { + self.print_tree_node(root, 0, &mut seen, filter, &mut out); + } + out + } + + fn print_tree_node( + &self, + key: &str, + depth: usize, + seen: &mut HashSet, + filter: Option<&HashSet>, + out: &mut String, + ) { + if !seen.insert(key.to_string()) { + return; + } + if let Some(dep) = self.nodes.get(key) { + let indent = " ".repeat(depth); + out.push_str(&format!("{}- {}\n", indent, dep.display_name)); + if let Some(children) = self.deps.get(key) { + for child in children.iter().sorted() { + if self.nodes.contains_key(child) + && filter.map_or(true, |f| f.contains(child)) + && !seen.contains(child) + { + self.print_tree_node(child, depth + 1, seen, filter, out); + } + } + } + } + } + + /// Produce topologically sorted layers. + pub fn layers(self) -> Vec>> { + let mut in_degree: HashMap = + self.nodes.keys().map(|k| (k.clone(), 0)).collect(); + let mut reverse: HashMap> = HashMap::new(); + + for (from, dep_set) in &self.deps { + for to in dep_set { + if self.nodes.contains_key(to) { + *in_degree.entry(from.clone()).or_default() += 1; + reverse.entry(to.clone()).or_default().push(from.clone()); + } + } + } + + let mut queue: VecDeque = in_degree + .iter() + .filter(|(_, &d)| d == 0) + .map(|(k, _)| k.clone()) + .sorted() + .collect(); + + let mut result = vec![]; + let mut nodes = self.nodes; + + while !queue.is_empty() { + let mut layer = vec![]; + let mut next = VecDeque::new(); + + for key in queue { + if let Some(dep) = nodes.remove(&key) { + layer.push(dep); + } + if let Some(dependents) = reverse.get(&key) { + for d in dependents { + if let Some(deg) = in_degree.get_mut(d) { + *deg -= 1; + if *deg == 0 { + next.push_back(d.clone()); + } + } + } + } + } + + if !layer.is_empty() { + result.push(layer); + } + queue = next.into_iter().sorted().collect(); + } + + result + } +} + +#[allow(dead_code)] +pub enum InstallDeps { + /// Flat list of dependencies — installed in one parallel batch (existing behavior). + Flat(Vec>), + /// Dependency graph — split into topological layers, each installed in parallel. + /// A `--- Layer N ---` separator is printed between layers. + Layered(DependencyGraph), +} + #[allow(dead_code)] pub enum InstallStrategy { /// Will invoke callback to install single dependency @@ -105,8 +253,10 @@ pub async fn par_install_language_dependencies_all_at_once< .await; } let total_time = std::time::Instant::now(); - let (missing, name_max_length) = filter_to_missing(deps, job_id, w_id, jailed, conn).await?; - if missing.is_empty() { + let (layers, name_max_length, total_missing) = + filter_to_missing(InstallDeps::Flat(deps), job_id, w_id, jailed, conn).await?; + let missing: Vec> = layers.into_iter().flatten().collect(); + if total_missing == 0 { return Ok(()); } let to_batch_install = Arc::new(RwLock::new(vec![])); @@ -122,6 +272,9 @@ pub async fn par_install_language_dependencies_all_at_once< conn, _language_name, _platform_agnostic, + None, + None, + None, ) .await?; let installation_res = process_handles(handles, w_id).await; @@ -231,18 +384,26 @@ pub async fn par_install_language_dependencies_seq< 'a, T: Clone + std::marker::Send + Sync + 'a + 'static, >( - deps: Vec>, + install_deps: InstallDeps, _language_name: &'a str, installer_executable_name: &'a str, _platform_agnostic: bool, concurrent_downloads: usize, callback: impl Fn(RequiredDependency) -> Result + Send + Sync + 'static, + post_install: Option) -> anyhow::Result<()> + Send + Sync + 'static>>, job_id: &'a Uuid, w_id: &'a str, worker_name: &'a str, jailed: bool, conn: &'a Connection, ) -> anyhow::Result<()> { + let total_time = std::time::Instant::now(); + let (layers, name_max_length, total_missing) = + filter_to_missing(install_deps, job_id, w_id, jailed, conn).await?; + if total_missing == 0 { + return Ok(()); + } + #[cfg(all(feature = "enterprise", feature = "parquet"))] let is_not_pro = !matches!( windmill_common::ee_oss::get_license_plan().await, @@ -258,65 +419,133 @@ pub async fn par_install_language_dependencies_seq< ) .await; } - let total_time = std::time::Instant::now(); - let (missing, name_max_length) = filter_to_missing(deps, job_id, w_id, jailed, conn).await?; - if missing.is_empty() { - return Ok(()); - } - let handles = spawn_wrapped_installation_threads( - missing, - name_max_length, - InstallStrategy::Single(Arc::new(callback)), - installer_executable_name, - concurrent_downloads, + + let is_layered = layers.len() > 1; + let callback = Arc::new(callback); + let mut offset = 0usize; + + windmill_queue::append_logs( job_id, w_id, - worker_name, + if jailed { + format!( + "\nStarting isolated installation... ({} tasks in parallel)\n", + concurrent_downloads + ) + } else { + format!( + "\nStarting installation... ({} tasks in parallel)\n", + concurrent_downloads + ) + }, conn, - _language_name, - _platform_agnostic, ) - .await?; + .await; + + for (i, layer_deps) in layers.into_iter().enumerate() { + if layer_deps.is_empty() { + continue; + } + + if is_layered && offset > 0 { + windmill_queue::append_logs( + job_id, + w_id, + format!("\n\n--- Layer {} ---", i + 1), + conn, + ) + .await; + } + + let layer_size = layer_deps.len(); + tracing::info!("Layer {}: spawning {} installs", i + 1, layer_size); + let handles = spawn_wrapped_installation_threads( + layer_deps, + name_max_length, + InstallStrategy::Single(callback.clone()), + installer_executable_name, + concurrent_downloads, + job_id, + w_id, + worker_name, + conn, + _language_name, + _platform_agnostic, + Some(offset), + Some(total_missing), + post_install.clone(), + ) + .await?; + tracing::info!("Layer {}: all spawned, waiting for handles", i + 1); + + process_handles(handles, w_id).await?; + tracing::info!("Layer {}: done", i + 1); + offset += layer_size; + } - let installation_res = process_handles(handles, w_id).await; finish_installation(total_time, job_id, w_id, conn).await; - installation_res + Ok(()) } type NameMaxLength = usize; + +/// Returns (layers of missing deps, name_max_length, total_missing). +/// Prints the "To be installed" header once with all missing packages. +/// For `Layered`, prints a dependency tree; for `Flat`, prints a flat list. async fn filter_to_missing<'a, T: Clone + std::marker::Send + Sync + 'a + 'static>( - mut deps: Vec>, + install_deps: InstallDeps, job_id: &Uuid, w_id: &str, jailed: bool, conn: &Connection, -) -> anyhow::Result<(Vec>, NameMaxLength)> { - // Unique to flatten all same values - deps = deps.into_iter().unique_by(|rd| rd.path.clone()).collect(); - // Total to install - let mut missing = vec![]; - // Name max length - let mut name_ml = 0; - for rd in deps.into_iter() { - let display_name = rd.display_name.clone(); - if rd.path.ends_with("/") { - anyhow::bail!("Internal error: path should not end with '/'") +) -> anyhow::Result<(Vec>>, NameMaxLength, usize)> { + let (mut layers, tree_data) = match install_deps { + InstallDeps::Flat(deps) => (vec![deps], None), + InstallDeps::Layered(graph) => { + let deps_map = graph.deps.clone(); + let nodes_display: HashMap = graph + .nodes + .iter() + .map(|(k, v)| (k.clone(), v.display_name.clone())) + .collect(); + let layers = graph.layers(); + (layers, Some((deps_map, nodes_display))) } - { - // Later will help us align text in log console - if display_name.len() > name_ml { + }; + + let mut name_ml = 0; + let mut missing_keys: HashSet = HashSet::new(); + let mut total_missing = 0; + + for layer in layers.iter_mut() { + *layer = std::mem::take(layer) + .into_iter() + .unique_by(|rd| rd.path.clone()) + .collect(); + + let mut missing = vec![]; + for rd in std::mem::take(layer) { + if rd.path.ends_with("/") { + anyhow::bail!("Internal error: path should not end with '/'") + } + if rd.display_name.len() > name_ml { name_ml = rd.display_name.len(); } + if tokio::fs::metadata(rd.path.clone() + ".valid.windmill") + .await + .is_err() + { + if let Some(key) = rd.path.rsplit('/').next() { + missing_keys.insert(key.to_string()); + } + missing.push(rd); + } } - // Will look like: /tmp/windmill/cache/lang/dependency.valid.windmill - if tokio::fs::metadata(rd.path.clone() + ".valid.windmill") - .await - .is_err() - { - missing.push(rd); - } + total_missing += missing.len(); + *layer = missing; } - if !missing.is_empty() { + + if total_missing > 0 { windmill_queue::append_logs( job_id, w_id, @@ -328,15 +557,40 @@ async fn filter_to_missing<'a, T: Clone + std::marker::Send + Sync + 'a + 'stati conn, ) .await; - let to_log = missing - .iter() - .map(|rd| format!("- {}", &rd.display_name)) - .join("\n") - + "\n"; + + let to_log = if let Some((deps_map, nodes_display)) = tree_data { + let mut print_graph: DependencyGraph<()> = DependencyGraph::new(); + for (key, display) in &nodes_display { + if missing_keys.contains(key) { + print_graph.insert( + key.clone(), + RequiredDependency { + path: String::new(), + _s3_handle: String::new(), + display_name: display.clone(), + custom_payload: (), + }, + deps_map + .get(key) + .map(|s| s.iter().cloned().collect()) + .unwrap_or_default(), + ); + } + } + print_graph.print_tree(Some(&missing_keys)) + } else { + layers + .iter() + .flat_map(|l| l.iter()) + .map(|rd| format!("- {}", &rd.display_name)) + .join("\n") + + "\n" + }; windmill_queue::append_logs(job_id, w_id, to_log, conn).await; } - Ok((missing, name_ml)) + + Ok((layers, name_ml, total_missing)) } enum Action { @@ -369,6 +623,9 @@ async fn spawn_wrapped_installation_threads< conn: &Connection, _language_name: &str, _platform_agnostic: bool, + counter_offset: Option, + total_override: Option, + post_install: Option) -> anyhow::Result<()> + Send + Sync + 'static>>, ) -> anyhow::Result<( Vec>>, tokio::sync::broadcast::Sender<()>, @@ -382,11 +639,11 @@ async fn spawn_wrapped_installation_threads< job_id ); - let (mut handles, semaphore, total_to_install, counter_arc) = ( + let total_to_install = total_override.unwrap_or(missing.len()); + let (mut handles, semaphore, counter_arc) = ( vec![], Arc::new(Semaphore::new(parallel_limit)), - missing.len(), - Arc::new(tokio::sync::Mutex::new(0)), + Arc::new(tokio::sync::Mutex::new(counter_offset.unwrap_or(0))), ); // Pretty sensitive. Single drop will fail installation @@ -426,6 +683,7 @@ async fn spawn_wrapped_installation_threads< ), InstallStrategy::AllAtOnce(ref rw_lock) => Action::AddToBulk(Arc::clone(rw_lock)), }; + let post_install_c = post_install.clone(); let task_fut = try_install_one_detached( dep, installer_executable_name.to_owned(), @@ -441,6 +699,7 @@ async fn spawn_wrapped_installation_threads< _platform_agnostic, permit, TaskKiller(kill_tx), + post_install_c, ); handles.push(tokio::spawn(async move { tokio::select! { @@ -513,6 +772,7 @@ async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a + // If dropped the entire installation fails and all installation threads are being stopped // That's why we just pass it to return so it is not being dropped kill_all_tasks: TaskKiller, + post_install: Option) -> anyhow::Result<()> + Send + Sync + 'static>>, ) -> anyhow::Result { let start = std::time::Instant::now(); @@ -607,6 +867,9 @@ async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a + &dep.display_name )); } else { + if let Some(ref cb) = post_install { + cb(&dep)?; + } mark_success(dep.path.clone(), &job_id, &w_id).await; print_success( false, diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs index 9b4ba3d92a..28de226102 100644 --- a/backend/windmill-worker/src/wac_executor.rs +++ b/backend/windmill-worker/src/wac_executor.rs @@ -364,6 +364,23 @@ pub fn is_wac_v2_ts(code: &str) -> bool { has_wac_import && has_workflow } +/// Inject the variable name as the first argument to `task()` calls in WAC v2 scripts. +/// `const double = task(async ...` → `const double = task("double", async ...` +/// Skips calls that already have a string argument. +pub fn inject_wac_task_names(content: &str) -> String { + use regex::Regex; + use std::borrow::Cow; + lazy_static::lazy_static! { + static ref TASK_RE: Regex = + Regex::new(r#"(?m)((?:export\s+)?(?:const|let|var)\s+)(\w+)(\s*=\s*task\s*(?:<[^>]*>)?\s*\(\s*)(async\b)"#).unwrap(); + } + let replaced = TASK_RE.replace_all(content, r#"${1}${2}${3}"${2}", ${4}"#); + match replaced { + Cow::Borrowed(_) => content.to_string(), + Cow::Owned(s) => s, + } +} + /// Detect WAC v2 patterns in Python code. /// Checks for `@workflow` decorator and `@task` decorator with wmill import, /// skipping comment lines. diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index b75403bef7..e40170dcd3 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -164,6 +164,9 @@ use crate::java_executor::{handle_java_job, JobHandlerInput as JobHandlerInputJa #[cfg(feature = "ruby")] use crate::ruby_executor::{handle_ruby_job, JobHandlerInput as JobHandlerInputRuby}; +#[cfg(feature = "rlang")] +use crate::r_executor::{handle_r_job, JobHandlerInput as JobHandlerInputRlang}; + #[cfg(feature = "php")] use crate::php_executor::handle_php_job; @@ -230,6 +233,9 @@ lazy_static::lazy_static! { // Ruby pub static ref RUBY_CACHE_DIR: String = format!("{}ruby", *ROOT_CACHE_DIR); + // R + pub static ref R_CACHE_DIR: String = format!("{}rlang", *ROOT_CACHE_DIR); + // for related places search: ADD_NEW_LANG pub static ref BUN_CACHE_DIR: String = format!("{}bun", *ROOT_CACHE_NOMOUNT_DIR); pub static ref BUN_BUNDLE_CACHE_DIR: String = format!("{}bun", *ROOT_CACHE_DIR); @@ -4602,7 +4608,8 @@ mount {{ | ScriptLang::Bash | ScriptLang::Powershell | ScriptLang::Ansible - | ScriptLang::Ruby => "#", + | ScriptLang::Ruby + | ScriptLang::Rlang => "#", ScriptLang::Deno | ScriptLang::Bun | ScriptLang::Bunnative @@ -5114,6 +5121,38 @@ mount {{ .await } } + ScriptLang::Rlang => { + #[cfg(not(feature = "rlang"))] + return Err( + anyhow::anyhow!("R is not available because the feature is not enabled").into(), + ); + + #[cfg(feature = "rlang")] + { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } + Box::pin(handle_r_job(JobHandlerInputRlang { + mem_peak, + canceled_by, + job, + conn, + client, + parent_runnable_path, + inner_content: &code, + job_dir, + requirements_o: lock.as_ref(), + shared_mount: &shared_mount, + base_internal_url, + worker_name, + envs, + occupancy_metrics, + })) + .await + } + } // for related places search: ADD_NEW_LANG _ => panic!("unreachable, language is not supported: {language:#?}"), }; @@ -5247,6 +5286,10 @@ pub fn parse_sig_of_lang( ScriptLang::Ruby => Some(windmill_parser_ruby::parse_ruby_signature(code)?), #[cfg(not(feature = "ruby"))] ScriptLang::Ruby => None, + #[cfg(feature = "rlang")] + ScriptLang::Rlang => Some(windmill_parser_r::parse_r_signature(code)?), + #[cfg(not(feature = "rlang"))] + ScriptLang::Rlang => None, // for related places search: ADD_NEW_LANG } } else { @@ -5412,3 +5455,4 @@ pub fn get_worker_internal_server_inline_utils( )), } } + diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 5ea5b74d64..dfdd06c2f1 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -61,6 +61,8 @@ use crate::csharp_executor::generate_nuget_lockfile; #[cfg(feature = "java")] use crate::java_executor; +#[cfg(feature = "rlang")] +use crate::r_executor; #[cfg(feature = "ruby")] use crate::ruby_executor; @@ -2763,6 +2765,21 @@ async fn capture_dependency_job( ) .await? } + #[cfg(feature = "rlang")] + ScriptLang::Rlang => { + r_executor::resolve( + job_id, + job_raw_code, + mem_peak, + canceled_by, + job_dir, + &Connection::Sql(db.clone()), + worker_name, + w_id, + false, + ) + .await? + } // for related places search: ADD_NEW_LANG _ => "".to_owned(), }; diff --git a/benchmarks/Dockerfile b/benchmarks/Dockerfile index c8f3fe83d5..7655e5f2d2 100644 --- a/benchmarks/Dockerfile +++ b/benchmarks/Dockerfile @@ -1,14 +1,20 @@ -FROM denoland/deno:alpine-1.26.2 +FROM denoland/deno:alpine-2.1.4 WORKDIR /app USER deno +ADD ./lib.ts . +ADD ./action.ts . ADD ./main.ts . -RUN deno cache --unstable main.ts +RUN deno cache main.ts ADD ./worker.ts . -RUN deno cache --unstable worker.ts +RUN deno cache worker.ts ADD ./scraper.ts . -RUN deno cache --unstable scraper.ts +RUN deno cache scraper.ts +ADD ./benchmark_oneoff.ts . +RUN deno cache benchmark_oneoff.ts +ADD ./benchmark_suite.ts . +RUN deno cache benchmark_suite.ts -ENTRYPOINT [ "/tini", "--", "docker-entrypoint.sh", "run", "--unstable", "-A", "main.ts" ] \ No newline at end of file +ENTRYPOINT [ "/tini", "--", "docker-entrypoint.sh", "run", "-A", "main.ts" ] diff --git a/benchmarks/README.md b/benchmarks/README.md index c358471626..fc758c006f 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,85 +1,71 @@ # Benchmarks -This folder includes a small deno/ts utility to benchmark execution of jobs & -flows. +Deno/TS benchmark suite for measuring Windmill job and flow execution throughput. -## Installation +## Quick Start -Install the `wmill` CLI tool using -`deno install --unstable -A https://deno.land/x/wmillbench/main.ts`. +```bash +# Install Deno +curl -fsSL https://deno.land/install.sh | sh -Update to the latest version using `wmillbench upgrade`. +# Run a single benchmark +deno run -A benchmark_oneoff.ts --kind noop --jobs 10000 -To build a local version, you can just run: -``` -deno install -A main.ts +# Run the full suite +deno run -A benchmark_suite.ts -c suite_config.json + +# Run WAC v2 benchmarks (workflow-as-code vs flow comparison) +deno run -A benchmark_suite.ts -c suite_wac.json ``` -## Quickstart +## Benchmark Kinds -Have your instance expose prometheus metrics (METRICS_ADDR=true). +### Script benchmarks +- `noop` — Empty jobs (measures pure scheduling overhead) +- `deno`, `bun`, `python`, `go`, `bash` — Language runtimes +- `nativets` — BunNative (no isolation) +- `dedicated`, `dedicated_nativets` — Dedicated worker mode -Then +### Flow benchmarks +- `2steps` — 2-step flow (deno + identity) +- `bigscriptinflow` — Flow with large raw bash script +- `flow_seq_2_bun` — 2 sequential bun steps +- `flow_par_2_bun` — 2 parallel bun steps (branchall) +- `flow_seq_3_bun` — 3 sequential bun steps +- `flow:` — Custom flow by path +- `script:` — Custom script by path -``` -wmillbench -e admin@windmill.dev -p changeme --host YOUR_HOST +### WAC v2 benchmarks (workflow-as-code) +- `wac_seq_2` — 2 sequential tasks +- `wac_par_2` — 2 parallel tasks (Promise.all) +- `wac_seq_3` — 3 sequential tasks +- `wac_inline_2` — 2 inline steps (no child jobs) + +## Suite Configs + +| File | Description | +|------|-------------| +| `suite_config.json` | Main benchmark suite (noop, languages, flows) | +| `suite_dedicated.json` | Dedicated worker benchmarks | +| `suite_dedicated_nativets.json` | Dedicated NativeTS benchmarks | +| `suite_wac.json` | WAC v2 vs flow comparison benchmarks | + +## Interactive Benchmark Tool + +```bash +deno run -A main.ts -e admin@windmill.dev -p changeme --host http://localhost:8000 ``` -## Usage +Options: `--workers`, `--seconds`, `--maximum-throughput`, `--use-flows`, `--script-pattern`, `--export-json`, `--export-csv` -Usage: wmillbench +## Graph Generation -Description: - -Run Benchmark to measure throughput of windmill. - -Options: - --h, --help - Show this help. --V, --version - Show the version number for this program. ---host - The windmill host to benchmark. (Default: "http://127.0.0.1:8000/") ---workers - The number of workers to run at once. (Default: 1) --s, --seconds - How long to run the benchmark for (in seconds). (Default: 30) --e, --email - The email to use to login. --p, --password - The password to use to login. --t, --token - The token to use when talking to the API server. Preferred over manual login. --w, --workspace - The workspace to spawn scripts from. (Default: "starter") --m, --metrics - The url to scrape metrics from. (Default: "http://localhost:8001/metrics") ---export-json - If set, exports will be into a JSON file. ---export-csv - If set, exports will be into a csv file. ---export-histograms [histograms...] - Mark metrics (without label) that are reported as histograms to export. ---export-simple [simple...] - Mark metrics (without label) that are reported as simple values. ---maximum-throughput - Maximum number of jobs/flows to start in one second. (Default: Infinity) ---use-flows - Run flows instead of jobs. ---histogram-buckets [buckets...] - Define what buckets to collect from histograms. (Default: [ "+Inf", "10", "5", "2.5", "2.5", "1", "0.5", "0.25", "0.1", "0.05", "0.025", "0.01", "0.005" ]) - -Environment variables: - -WM_TOKEN - The token to use when talking to the API server. Preferred -over manual login. WM_WORKSPACE - The workspace to spawn scripts -from. - - - -This will run a simple benchmark against localhost (the default admin email + -password are set above), all execution is done in the "bench" workspace (as set -via `--workspace`). - -Metrics are exported to JSON will only include mean & stdev, histograms get one -entry for each bucket. CSV will include a full list of all values scraped. - -## NOOP jobs benchmark - -A specific benchmark creating a set of NOOP jobs all at once in windmill is also available. -in `benchmarks_noop.ts` - -You can build it locally with: -``` -deno install -A benchmarks_noop.ts -``` -and then -``` -benchmarks_noop -e admin@windmill.dev -p changeme --host YOUR_HOST +```bash +deno run -A benchmark_graphs.ts -c graphs_config.json ``` -By default it creates 10000 jobs in Windmill in a single batch, but this is parametrizable. \ No newline at end of file +Generates SVG graphs from `*_benchmark.json` data files. + +## CI + +The GitHub Actions workflow (`.github/workflows/benchmark.yml`) runs hourly with 1/4/8 worker configurations plus WAC benchmarks. Results are committed to the `benchmarks` branch. diff --git a/benchmarks/benchmark_graphs.ts b/benchmarks/benchmark_graphs.ts index 23b3b763f7..bfaad7dbb6 100644 --- a/benchmarks/benchmark_graphs.ts +++ b/benchmarks/benchmark_graphs.ts @@ -3,32 +3,20 @@ import { UpgradeCommand } from "https://deno.land/x/cliffy@v0.25.7/command/upgra import { DenoLandProvider } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts"; import { drawGraph, drawGraphMulti } from "./graph.ts"; -import { VERSION } from "./lib.ts"; +import { VERSION, loadJsonConfig } from "./lib.ts"; -type GraphsConfig = [ - { - graph_title: string; - benchmarks: { - kind: string; - workers: number; - label: string; - }[]; - jobs: number; - } -]; +type GraphsConfig = { + graph_title: string; + benchmarks: { + kind: string; + workers: number; + label: string; + }[]; +}[]; async function main({ configPath }: { configPath: string }) { - async function getConfig(configPath: string): Promise { - if (configPath.startsWith("http")) { - const response = await fetch(configPath); - return await response.json(); - } else { - return JSON.parse(await Deno.readTextFile(configPath)); - } - } - try { - const config = await getConfig(configPath); + const config = await loadJsonConfig(configPath); for (const graphConfig of config || []) { const data: { @@ -81,7 +69,7 @@ async function main({ configPath }: { configPath: string }) { } await new Command() - .name("wmillbenchsuite") + .name("wmillbenchgraphs") .description("Create and save graphs from benchmark data.") .version(VERSION) .option("-c --config-path ", "The path of the config file", { diff --git a/benchmarks/benchmark_oneoff.ts b/benchmarks/benchmark_oneoff.ts index 0cd4c3483f..5f075d2034 100644 --- a/benchmarks/benchmark_oneoff.ts +++ b/benchmarks/benchmark_oneoff.ts @@ -10,7 +10,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -import { VERSION, createBenchScript, getFlowPayload, login } from "./lib.ts"; +import { VERSION, createBenchScript, createWacBenchScript, getFlowPayload, login, WAC_KINDS, STEPS_PER_WORKFLOW } from "./lib.ts"; async function verifyOutputs(uuids: string[], workspace: string) { console.log("Verifying outputs"); @@ -38,6 +38,8 @@ async function verifyOutputs(uuids: string[], workspace: string) { } export const NON_TEST_TAGS = ["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets", "flow"] + +const FLOW_COMPARISON_KINDS = ["flow_seq_2_bun", "flow_par_2_bun", "flow_seq_3_bun"]; export async function main({ host, email, @@ -151,6 +153,8 @@ export async function main({ ) ) { await createBenchScript(kind, workspace); + } else if (WAC_KINDS.includes(kind)) { + await createWacBenchScript(kind, workspace); } @@ -173,6 +177,20 @@ export async function main({ kind: "script", path: "f/benchmarks/" + kind, }); + } else if (WAC_KINDS.includes(kind)) { + // WAC v2 scripts are deployed as bun scripts, run via script path + nStepsFlow = STEPS_PER_WORKFLOW[kind] ?? 0; + body = JSON.stringify({ + kind: "script", + path: "f/benchmarks/" + kind, + }); + } else if (FLOW_COMPARISON_KINDS.includes(kind)) { + nStepsFlow = STEPS_PER_WORKFLOW[kind] ?? 0; + const payload = getFlowPayload(kind); + body = JSON.stringify({ + kind: "flow", + flow_value: payload.value, + }); } else if (["2steps", "bigscriptinflow"].includes(kind)) { nStepsFlow = kind == "2steps" ? 2 : 1; const payload = getFlowPayload(kind); @@ -182,7 +200,7 @@ export async function main({ }); } else if (kind.startsWith("flow:")) { console.log("Detected custom flow "); - let flow_path = kind.substr(5); + let flow_path = kind.substring(5); nStepsFlow = await getFlowStepCount(config.workspace_id, flow_path); console.log(`Total steps of flow including sub-flows: ${nStepsFlow}`); body = JSON.stringify({ @@ -193,7 +211,7 @@ export async function main({ console.log("Detected custom script"); body = JSON.stringify({ kind: "script", - path: kind.substr(7), + path: kind.substring(7), }); } else if (kind == "bigrawscript") { noVerify = true; @@ -281,6 +299,9 @@ export async function main({ let lastElapsed = 0; let lastCompletedJobs = 0; + // Timeout: 10 minutes for the polling loop to prevent hanging forever + // (e.g. if WAC suspend/resume fails or jobs get stuck) + const POLL_TIMEOUT_MS = 10 * 60 * 1000; let didStart = false; while (completedJobs < jobsSent) { const loopStart = Date.now(); @@ -292,6 +313,10 @@ export async function main({ } } else { const elapsed = start ? Date.now() - start : 0; + if (elapsed > POLL_TIMEOUT_MS) { + console.error(`\nTimeout: benchmark did not complete within ${POLL_TIMEOUT_MS / 1000}s (${completedJobs}/${jobsSent} completed)`); + break; + } completedJobs = await getCompletedJobsCount(NON_TEST_TAGS); if (nStepsFlow > 0) { completedJobs = Math.floor(completedJobs / (nStepsFlow + 1)); @@ -338,7 +363,9 @@ export async function main({ kind !== "nativets" && kind !== "dedicated_nativets" && !kind.startsWith("flow:") && - !kind.startsWith("script:") + !kind.startsWith("script:") && + !WAC_KINDS.includes(kind) && + !FLOW_COMPARISON_KINDS.includes(kind) ) { await verifyOutputs(uuids, config.workspace_id); } @@ -387,7 +414,7 @@ if (import.meta.main) { ) .option( "--kind ", - "Specifiy the benchmark kind among: deno, identity, python, go, bash, dedicated, bun, noop, 2steps, nativets, dedicated_nativets", + "Specifiy the benchmark kind among: deno, identity, python, go, bash, dedicated, bun, noop, 2steps, nativets, dedicated_nativets, wac_seq_2, wac_par_2, wac_seq_3, wac_inline_2, flow_seq_2_bun, flow_par_2_bun, flow_seq_3_bun", { required: true, } diff --git a/benchmarks/benchmark_suite.ts b/benchmarks/benchmark_suite.ts index f4840dda05..caba48aa72 100644 --- a/benchmarks/benchmark_suite.ts +++ b/benchmarks/benchmark_suite.ts @@ -4,7 +4,7 @@ import { DenoLandProvider } from "https://deno.land/x/cliffy@v0.25.7/command/upg import { main as runBenchmark } from "./benchmark_oneoff.ts"; -import { VERSION } from "./lib.ts"; +import { VERSION, loadJsonConfig } from "./lib.ts"; type Config = { kind: string; @@ -50,21 +50,12 @@ async function main({ workers: number; factor?: number; }) { - async function getConfig(configPath: string): Promise { - if (configPath.startsWith("http")) { - const response = await fetch(configPath); - return await response.json(); - } else { - return JSON.parse(await Deno.readTextFile(configPath)); - } - } - if (!Deno.args.includes("--no-warm-up")) { await warmUp(host, email, password, token, workspace); } try { - const config = await getConfig(configPath); + const config = await loadJsonConfig(configPath); for (const benchmark of config) { try { console.log( diff --git a/benchmarks/graphs_config.json b/benchmarks/graphs_config.json index 174990ecb2..8e37695d66 100644 --- a/benchmarks/graphs_config.json +++ b/benchmarks/graphs_config.json @@ -223,5 +223,75 @@ "label": "noop" } ] + }, + { + "graph_title": "WAC v2 sequential vs flow sequential (2 steps, bun)", + "benchmarks": [ + { + "kind": "wac_seq_2", + "workers": 1, + "label": "WAC v2 sequential" + }, + { + "kind": "flow_seq_2_bun", + "workers": 1, + "label": "Flow sequential" + } + ] + }, + { + "graph_title": "WAC v2 parallel vs flow parallel (2 steps, bun)", + "benchmarks": [ + { + "kind": "wac_par_2", + "workers": 1, + "label": "WAC v2 parallel" + }, + { + "kind": "flow_par_2_bun", + "workers": 1, + "label": "Flow parallel" + } + ] + }, + { + "graph_title": "WAC v2 sequential vs flow sequential (3 steps, bun)", + "benchmarks": [ + { + "kind": "wac_seq_3", + "workers": 1, + "label": "WAC v2 sequential" + }, + { + "kind": "flow_seq_3_bun", + "workers": 1, + "label": "Flow sequential" + } + ] + }, + { + "graph_title": "WAC v2 patterns comparison", + "benchmarks": [ + { + "kind": "wac_seq_2", + "workers": 1, + "label": "sequential 2-task" + }, + { + "kind": "wac_par_2", + "workers": 1, + "label": "parallel 2-task" + }, + { + "kind": "wac_seq_3", + "workers": 1, + "label": "sequential 3-task" + }, + { + "kind": "wac_inline_2", + "workers": 1, + "label": "inline 2-step" + } + ] } ] \ No newline at end of file diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 9af7dc3af2..3b9863c17e 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.665.0"; +export const VERSION = "v1.672.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ @@ -132,6 +132,119 @@ export async function createBenchScript( } } +// WAC v2 benchmark script content patterns +const WAC_SCRIPTS: Record = { + wac_seq_2: [ + 'import { task, workflow } from "windmill-client";', + "const step_a = task(async () => { return 1; });", + "const step_b = task(async () => { return 2; });", + "export const main = workflow(async () => {", + " const a = await step_a();", + " const b = await step_b();", + " return { a, b };", + "});", + ].join("\n"), + + wac_par_2: [ + 'import { task, workflow } from "windmill-client";', + "const step_a = task(async () => { return 1; });", + "const step_b = task(async () => { return 2; });", + "export const main = workflow(async () => {", + " const [a, b] = await Promise.all([step_a(), step_b()]);", + " return { a, b };", + "});", + ].join("\n"), + + wac_seq_3: [ + 'import { task, workflow } from "windmill-client";', + "const step_a = task(async () => { return 1; });", + "const step_b = task(async () => { return 2; });", + "const step_c = task(async () => { return 3; });", + "export const main = workflow(async () => {", + " const a = await step_a();", + " const b = await step_b();", + " const c = await step_c();", + " return { a, b, c };", + "});", + ].join("\n"), + + wac_inline_2: [ + 'import { step, workflow } from "windmill-client";', + "export const main = workflow(async () => {", + ' const a = await step("a", () => 1);', + ' const b = await step("b", () => 2);', + " return { a, b };", + "});", + ].join("\n"), +}; + +export const WAC_KINDS = Object.keys(WAC_SCRIPTS); + +// Number of child jobs created per workflow instance (used to compute throughput) +// For task(): each task creates a child job. For step(): no child job. +// Total completed jobs per workflow = nSteps + 1 (children + parent) +export const STEPS_PER_WORKFLOW: Record = { + wac_seq_2: 2, + wac_par_2: 2, + wac_seq_3: 3, + wac_inline_2: 0, // inline steps don't create child jobs + flow_seq_2_bun: 2, + flow_par_2_bun: 2, + flow_seq_3_bun: 3, +}; + +export async function createWacBenchScript( + wacPattern: string, + workspace: string, +) { + const scriptContent = WAC_SCRIPTS[wacPattern]; + if (!scriptContent) { + throw new Error("Unknown WAC pattern: " + wacPattern); + } + + const path = `f/benchmarks/${wacPattern}`; + const exists = await windmill.ScriptService.existsScriptByPath({ + workspace, + path, + }); + + if (exists) { + await windmill.ScriptService.deleteScriptByPath({ + workspace, + path, + }); + } + + const hash = await windmill.ScriptService.createScript({ + workspace, + requestBody: { + path, + content: scriptContent, + summary: wacPattern + " WAC v2 benchmark", + description: "", + language: "bun" as api.NewScript.language, + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + properties: {}, + required: [], + type: "object", + }, + }, + }); + + await waitForDeployment(workspace, hash); + console.log("Created WAC v2 benchmark script at path", path); +} + +export async function loadJsonConfig(configPath: string): Promise { + if (configPath.startsWith("http")) { + const response = await fetch(configPath); + return await response.json(); + } else { + return JSON.parse(await Deno.readTextFile(configPath)); + } +} + export const getFlowPayload = (flowPattern: string): api.FlowPreview => { if (flowPattern == "branchone") { return { @@ -260,6 +373,113 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => { ], }, }; + } else if (flowPattern == "flow_seq_2_bun") { + return { + path: "flow_seq_2_bun", + args: {}, + value: { + modules: [ + { + id: "a", + value: { + input_transforms: {}, + language: "bun" as api.RawScript.language, + type: "rawscript", + content: "export function main() { return 1; }", + }, + }, + { + id: "b", + value: { + input_transforms: {}, + language: "bun" as api.RawScript.language, + type: "rawscript", + content: "export function main() { return 2; }", + }, + }, + ], + }, + }; + } else if (flowPattern == "flow_par_2_bun") { + return { + path: "flow_par_2_bun", + args: {}, + value: { + modules: [ + { + id: "a", + value: { + type: "branchall", + parallel: true, + branches: [ + { + modules: [ + { + id: "b", + value: { + input_transforms: {}, + language: "bun" as api.RawScript.language, + type: "rawscript", + content: "export function main() { return 1; }", + }, + }, + ], + }, + { + modules: [ + { + id: "c", + value: { + input_transforms: {}, + language: "bun" as api.RawScript.language, + type: "rawscript", + content: "export function main() { return 2; }", + }, + }, + ], + }, + ], + }, + }, + ], + }, + }; + } else if (flowPattern == "flow_seq_3_bun") { + return { + path: "flow_seq_3_bun", + args: {}, + value: { + modules: [ + { + id: "a", + value: { + input_transforms: {}, + language: "bun" as api.RawScript.language, + type: "rawscript", + content: "export function main() { return 1; }", + }, + }, + { + id: "b", + value: { + input_transforms: {}, + language: "bun" as api.RawScript.language, + type: "rawscript", + content: "export function main() { return 2; }", + }, + }, + { + id: "c", + value: { + input_transforms: {}, + language: "bun" as api.RawScript.language, + type: "rawscript", + content: "export function main() { return 3; }", + }, + }, + ], + }, + }; } else { return { path: "2steps", diff --git a/benchmarks/main.ts b/benchmarks/main.ts index 4d8d144cd4..1f681a9c03 100644 --- a/benchmarks/main.ts +++ b/benchmarks/main.ts @@ -264,24 +264,6 @@ export async function main({ ); const shutdown_start = Date.now(); - // let zombie_jobs = 0; - // let incorrect_results = 0; - // workers.forEach((worker, i) => { - // const l = (evt: MessageEvent) => { - // if (evt.data.type === "zombie_jobs") { - // zombie_jobs += evt.data.zombie_jobs; - // incorrect_results += evt.data.incorrect_results; - // worker.removeEventListener("message", l); - // workers = workers.filter((w) => w != worker); - // jobsSent[i] = evt.data.jobs_sent; - // worker.terminate(); - // } - // }; - // worker.addEventListener("message", l); - // worker.postMessage( - // Number.isSafeInteger(zombieTimeout) ? zombieTimeout : 90000 - // ); - // }); workers.forEach((worker, i) => { const l = (evt: MessageEvent) => { if (evt.data.type === "done") { @@ -327,8 +309,6 @@ export async function main({ console.log("time (s + tts):", time); console.log("throughput /s (jobs/time):", sum / time); - // console.log("zombie jobs: ", zombie_jobs); - // console.log("incorrect results: ", incorrect_results); console.log( "queue length:", ( diff --git a/benchmarks/suite_wac.json b/benchmarks/suite_wac.json new file mode 100644 index 0000000000..e677952761 --- /dev/null +++ b/benchmarks/suite_wac.json @@ -0,0 +1,30 @@ +[ + { + "kind": "wac_seq_2", + "jobs": 250 + }, + { + "kind": "wac_par_2", + "jobs": 250 + }, + { + "kind": "wac_seq_3", + "jobs": 200 + }, + { + "kind": "wac_inline_2", + "jobs": 500 + }, + { + "kind": "flow_seq_2_bun", + "jobs": 250 + }, + { + "kind": "flow_par_2_bun", + "jobs": 250 + }, + { + "kind": "flow_seq_3_bun", + "jobs": 200 + } +] diff --git a/benchmarks/worker.ts b/benchmarks/worker.ts index cd56d0fb45..ae06fa9779 100644 --- a/benchmarks/worker.ts +++ b/benchmarks/worker.ts @@ -139,96 +139,6 @@ while (cont) { clearInterval(updateStatusInterval); -// const end_time = Date.now() + complete_timeout; - -// let incorrect_results = 0; -// const enc = (s: string) => new TextEncoder().encode(s); - -// let last_queue_length = await getQueueCount(); -// console.log(`waiting for ${last_queue_length} jobs to complete...`); - -// while ( -// outstanding.length > 0 && -// last_queue_length > 0 && -// Date.now() < end_time -// ) { -// try { -// if (!config.hideProgress) { -// await Deno.stdout.write( -// enc( -// "\rwaiting for jobs to complete: outstanding " + -// outstanding.length + -// " - queue" + -// last_queue_length + -// "\n" -// ) -// ); -// } -// last_queue_length = await getQueueCount(); - -// const uuid = outstanding.shift()!; - -// let r: Job; -// try { -// r = await windmill.JobService.getJob({ -// workspace: config.workspace_id, -// id: uuid, -// }); -// } catch (e) { -// console.log("job not found: " + uuid + " " + e.message); -// continue; -// } -// if (r.type == "QueuedJob") { -// outstanding.push(uuid); - -// if (!config.hideProgress) { -// await Deno.stdout.write( -// enc(`uuid: ${uuid}, queue length: ${last_queue_length}\r`) -// ); -// } -// } else { -// r = r as api.CompletedJob; -// try { -// if ( -// ![ -// "httpversion", -// "identity", -// "httpslow", -// "noop", -// "dedicated", -// ].includes(config.scriptPattern) && -// r.result != uuid -// ) { -// console.log( -// "job did not return correct UUID: " + -// r.result + -// " != " + -// uuid + -// "job: \n" + -// JSON.stringify(r, null, 2) -// ); -// incorrect_results++; -// } else { -// // console.log(r.result); -// } -// } catch (e) { -// console.log("error during wait: ", e); -// outstanding.push(uuid); -// } -// } -// } catch (e) { -// console.log("error while waiting for outstanding jobs, sleeing: ", e); -// await sleep(0.5); -// } -// } - -// self.postMessage({ -// type: "zombie_jobs", -// zombie_jobs: outstanding.length, -// incorrect_results, -// jobs_sent: total_spawned, -// }); - self.postMessage({ type: "done", jobs_sent: total_spawned, diff --git a/cli/TESTING.md b/cli/TESTING.md index 9928e75266..542baab368 100644 --- a/cli/TESTING.md +++ b/cli/TESTING.md @@ -3,57 +3,57 @@ ## Running Tests ```bash -# Run all tests -deno test -A --no-check test/ +# Run unit tests only (fast — no backend, no database, no cargo build) +bun run test:unit + +# Run all tests (unit + integration — requires PostgreSQL + cargo) +DATABASE_URL=postgres://postgres:changeme@localhost:5432 bun run test # Run specific test files -deno test -A --no-check test/gitsync_settings_features.test.ts -deno test -A --no-check test/init_no_git_sync.test.ts -deno test -A --no-check test/multi_instance_workspace.test.ts -deno test -A --no-check test/override_settings_behavior.test.ts -deno test -A --no-check test/sync_config_resolution.test.ts -deno test -A --no-check test/workspace_conflicts.test.ts - -# Run with specific test patterns -deno test -A --no-check test/ --filter "workspace" -deno test -A --no-check test/ --filter "sync" +bun test test/sync_pull_push.test.ts +bun test test/workspace_conflicts_unit.test.ts ``` -## Test Files +## Test Categories -- **`gitsync_settings_features.test.ts`** - Git sync settings functionality -- **`init_no_git_sync.test.ts`** - Init without git sync -- **`multi_instance_workspace.test.ts`** - Multi-instance workspace handling -- **`override_settings_behavior.test.ts`** - Settings override behavior -- **`sync_config_resolution.test.ts`** - Sync configuration resolution -- **`workspace_conflicts.test.ts`** - Workspace conflict detection +### Unit tests (`*_unit.test.ts`) -## Docker Requirements +Pure local tests — no backend, no database. Uses `bunfig.unit.toml` (no preload). + +Examples: `git_unit`, `lint_command_unit`, `tar_creation_unit`, `workspace_conflicts_unit` + +### Integration tests + +Require a running backend and PostgreSQL. The `setup.ts` preload builds the backend +binary and starts a shared backend instance. + +Examples: `sync_pull_push`, `dev_server`, `standalone_commands` + +## Environment Variables + +| Variable | Purpose | Default | +|----------|---------|---------| +| `DATABASE_URL` | PostgreSQL connection string (without database name) | `postgres://postgres:changeme@localhost:5432` | +| `TEST_BACKEND` | `cargo` or `docker` | `cargo` | +| `CI_MINIMAL_FEATURES` | `true` for CI mode (zip-only features) | unset | +| `EE_LICENSE_KEY` | Enterprise license for EE feature tests | unset | +| `TEST_FEATURES` | Additional cargo features (comma-separated) | unset | +| `TEST_CLI_RUNTIME` | `node` to test npm package | unset | +| `UNIT_ONLY` | `1` to skip backend setup in preload (used by `test:unit`) | unset | +| `VERBOSE` | `1` for backend process output | unset | + +## Cleanup + +Stale test databases (`windmill_test_*`) and orphaned backend processes from +previous crashed runs are automatically cleaned up when starting a new test run. + +To manually check for leftovers: ```bash -# Ensure Docker is running -docker --version -docker-compose --version +# Check for stale test databases +psql postgres://postgres:changeme@localhost:5432/postgres -c \ + "SELECT datname FROM pg_database WHERE datname LIKE 'windmill_test_%';" -# Ensure EE license key is available -echo $EE_LICENSE_KEY +# Check for orphaned backend processes +ps aux | grep "target/debug/windmill" | grep -v grep ``` - -## Debugging Failed Tests - -```bash -# Run with verbose output -deno test -A --no-check test/ --reporter=verbose - -# Check container status -docker ps - -# View backend logs -docker logs test-test_windmill_server-1 - -# Manual container management -cd test -docker compose -f docker-compose.test.yml up -d -docker compose -f docker-compose.test.yml down -docker compose -f docker-compose.test.yml down -v -``` \ No newline at end of file diff --git a/cli/bootstrap/flow_bootstrap.ts b/cli/bootstrap/flow_bootstrap.ts index 8bae373b17..3a71051505 100644 --- a/cli/bootstrap/flow_bootstrap.ts +++ b/cli/bootstrap/flow_bootstrap.ts @@ -13,7 +13,6 @@ export interface FlowDefinition { properties: { [name: string]: SchemaProperty}, required: string[] } - ws_error_handler_muted: false } export function defaultFlowDefinition(): FlowDefinition { @@ -30,6 +29,5 @@ export function defaultFlowDefinition(): FlowDefinition { properties: {}, required: [] }, - ws_error_handler_muted: false, } } diff --git a/cli/bootstrap/script_bootstrap.ts b/cli/bootstrap/script_bootstrap.ts index 89d98a1927..43dabca093 100644 --- a/cli/bootstrap/script_bootstrap.ts +++ b/cli/bootstrap/script_bootstrap.ts @@ -134,6 +134,11 @@ public class Main { def main a, b, c puts a, b, c end +`, + rlang: ` +main <- function(x, name = "default") { + return(list(result = x, name = name)) +} `, // for related places search: ADD_NEW_LANG }; diff --git a/cli/generate-schema.ts b/cli/generate-schema.ts new file mode 100644 index 0000000000..a5925968d6 --- /dev/null +++ b/cli/generate-schema.ts @@ -0,0 +1,16 @@ +#!/usr/bin/env npx tsx +/** + * Regenerate cli/wmill.schema.json from CONFIG_REFERENCE. + * + * Run after adding or modifying config options in src/commands/init/template.ts: + * npx tsx generate-schema.ts + */ +import { writeFileSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { generateJsonSchema } from "./src/commands/init/template.ts"; + +const dir = dirname(fileURLToPath(import.meta.url)); +const out = join(dir, "wmill.schema.json"); +writeFileSync(out, JSON.stringify(generateJsonSchema(), null, 2) + "\n"); +console.log(`Wrote ${out}`); diff --git a/cli/package.json b/cli/package.json index e44a215631..105915a720 100644 --- a/cli/package.json +++ b/cli/package.json @@ -9,6 +9,7 @@ "dev": "bun run src/main.ts", "build": "./build.sh", "test": "bun test test/", + "test:unit": "UNIT_ONLY=1 bun test test/*_unit*", "check": "bunx tsc --noEmit", "gen-client": "./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh" }, diff --git a/cli/src/commands/app/app.ts b/cli/src/commands/app/app.ts index f048aac616..13851e5f8c 100644 --- a/cli/src/commands/app/app.ts +++ b/cli/src/commands/app/app.ts @@ -5,6 +5,7 @@ import { Table } from "@cliffy/table"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; import { sep as SEP } from "node:path"; +import { stat } from "node:fs/promises"; import * as windmillUtils from "@windmill-labs/shared-utils"; import { yamlParseFile } from "../../utils/yaml.ts"; import * as wmill from "../../../gen/services.gen.ts"; @@ -241,8 +242,26 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) { const workspace = await resolveWorkspace(opts); await requireLogin(opts); - await pushApp(workspace.workspaceId, remotePath, filePath); - log.info(colors.bold.underline.green("App pushed")); + // Detect raw apps by checking for raw_app.yaml or __raw_app/.raw_app suffix + const normalizedPath = filePath.endsWith(SEP) ? filePath.slice(0, -1) : filePath; + const isRawApp = normalizedPath.endsWith("__raw_app") || normalizedPath.endsWith(".raw_app"); + let hasRawAppYaml = false; + if (!isRawApp) { + try { + const rawAppPath = (filePath.endsWith(SEP) ? filePath : filePath + SEP) + "raw_app.yaml"; + await stat(rawAppPath); + hasRawAppYaml = true; + } catch { /* not a raw app */ } + } + + if (isRawApp || hasRawAppYaml) { + const { pushRawApp } = await import("./raw_apps.ts"); + await pushRawApp(workspace.workspaceId, remotePath, filePath); + log.info(colors.bold.underline.green("Raw app pushed")); + } else { + await pushApp(workspace.workspaceId, remotePath, filePath); + log.info(colors.bold.underline.green("App pushed")); + } } const command = new Command() diff --git a/cli/src/commands/app/bundle.ts b/cli/src/commands/app/bundle.ts index 998546d364..8d15488491 100644 --- a/cli/src/commands/app/bundle.ts +++ b/cli/src/commands/app/bundle.ts @@ -166,8 +166,10 @@ export async function createBundle( // Dynamically import esbuild const esbuild = await import("esbuild"); - // Detect frameworks to determine default entry point - const frameworks = detectFrameworks(process.cwd()); + // Detect frameworks to determine default entry point. + // Use the entryPoint's directory if provided, otherwise fall back to cwd. + const appDir = options.entryPoint ? path.dirname(options.entryPoint) : process.cwd(); + const frameworks = detectFrameworks(appDir); const defaultEntry = (frameworks.svelte || frameworks.vue) ? "index.ts" : "index.tsx"; const entryPoint = options.entryPoint ?? defaultEntry; @@ -184,7 +186,6 @@ export async function createBundle( } // Ensure node_modules exists in the app directory - const appDir = path.dirname(entryPoint) || process.cwd(); await ensureNodeModules(appDir); // Load framework-specific plugins (svelte, vue) based on package.json diff --git a/cli/src/commands/app/lint.ts b/cli/src/commands/app/lint.ts index 12014cc7d5..9356b5dd3c 100644 --- a/cli/src/commands/app/lint.ts +++ b/cli/src/commands/app/lint.ts @@ -6,7 +6,7 @@ import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; import { yamlParseFile } from "../../utils/yaml.ts"; import { GlobalOptions } from "../../types.ts"; -import { createBundle } from "./bundle.ts"; +import { createBundle, detectFrameworks } from "./bundle.ts"; import { APP_BACKEND_FOLDER } from "./app_metadata.ts"; import { loadRunnablesFromBackend } from "./raw_apps.ts"; import { @@ -113,7 +113,11 @@ async function validateBuild( log.info(colors.blue("🔨 Testing build...")); // Try to create a bundle - this will validate that all dependencies are in place + const frameworks = detectFrameworks(appDir); + const entryFile = frameworks.svelte || frameworks.vue ? "index.ts" : "index.tsx"; + const entryPoint = path.join(appDir, entryFile); await createBundle({ + entryPoint, production: true, minify: false, }); diff --git a/cli/src/commands/audit/audit.ts b/cli/src/commands/audit/audit.ts new file mode 100644 index 0000000000..e0a85e89b6 --- /dev/null +++ b/cli/src/commands/audit/audit.ts @@ -0,0 +1,120 @@ +import { GlobalOptions } from "../../types.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { Command } from "@cliffy/command"; +import { Table } from "@cliffy/table"; +import { colors } from "@cliffy/ansi/colors"; +import * as log from "../../core/log.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import { formatTimestamp } from "../../utils/utils.ts"; + +async function list( + opts: GlobalOptions & { + json?: boolean; + username?: string; + operation?: string; + actionKind?: string; + before?: string; + after?: string; + limit?: number; + } +) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const logs = await wmill.listAuditLogs({ + workspace: workspace.workspaceId, + username: opts.username, + operation: opts.operation, + actionKind: opts.actionKind as any, + before: opts.before, + after: opts.after, + perPage: opts.limit ?? 30, + }); + + if (opts.json) { + console.log(JSON.stringify(logs)); + } else { + if (logs.length === 0) { + log.info("No audit logs found."); + return; + } + if (logs.every((l) => l.operation === "redacted")) { + log.info(colors.yellow( + "Audit log details are not available on the Community Edition.\n" + + "Upgrade to the Enterprise Edition for full audit logging with operation details." + )); + return; + } + new Table() + .header(["ID", "Timestamp", "Username", "Operation", "Action", "Resource"]) + .padding(2) + .border(true) + .body( + logs.map((l) => [ + String(l.id), + formatTimestamp(l.timestamp), + l.username, + l.operation, + l.action_kind, + l.resource ?? "-", + ]) + ) + .render(); + } +} + +async function get( + opts: GlobalOptions & { json?: boolean }, + id: string +) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const auditLog = await wmill.getAuditLog({ + workspace: workspace.workspaceId, + id: parseInt(id, 10), + }); + + if (opts.json) { + console.log(JSON.stringify(auditLog)); + } else { + console.log(colors.bold("ID:") + " " + auditLog.id); + console.log(colors.bold("Timestamp:") + " " + formatTimestamp(auditLog.timestamp)); + console.log(colors.bold("Username:") + " " + auditLog.username); + console.log(colors.bold("Operation:") + " " + auditLog.operation); + console.log(colors.bold("Action Kind:") + " " + auditLog.action_kind); + console.log(colors.bold("Resource:") + " " + (auditLog.resource ?? "-")); + if (auditLog.parameters && Object.keys(auditLog.parameters).length > 0) { + console.log(colors.bold("Parameters:")); + console.log(JSON.stringify(auditLog.parameters, null, 2)); + } + } +} + +const auditListOptions = (cmd: Command) => + cmd + .option("--json", "Output as JSON (for piping to jq)") + .option("--username ", "Filter by username") + .option("--operation ", "Filter by operation (exact or prefix)") + .option("--action-kind ", "Filter by action kind (Create, Update, Delete, Execute)") + .option("--before ", "Filter events before this timestamp") + .option("--after ", "Filter events after this timestamp") + .option("--limit ", "Number of entries to return (default 30, max 100)"); + +const command = auditListOptions(new Command() + .description("View audit logs (requires admin)")) + .action(list as any) + .command("list", auditListOptions(new Command().description("List audit log entries"))) + .action(list as any) + .command("get", "Get a specific audit log entry") + .arguments("") + .option("--json", "Output as JSON (for piping to jq)") + .action(get as any); + +export default command; diff --git a/cli/src/commands/config/config.ts b/cli/src/commands/config/config.ts new file mode 100644 index 0000000000..500e65a113 --- /dev/null +++ b/cli/src/commands/config/config.ts @@ -0,0 +1,26 @@ +import { Command } from "@cliffy/command"; +import * as log from "../../core/log.ts"; +import { + formatConfigReference, + formatConfigReferenceJson, +} from "../init/template.ts"; + +interface ConfigOptions { + json?: boolean; +} + +async function configAction(opts: ConfigOptions) { + if (opts.json) { + console.log(formatConfigReferenceJson()); + } else { + log.info(formatConfigReference()); + } +} + +const command = new Command() + .name("config") + .description("Show all available wmill.yaml configuration options") + .option("--json", "Output as JSON for programmatic consumption") + .action(configAction as any); + +export default command; diff --git a/cli/src/commands/dependencies/dependencies.ts b/cli/src/commands/dependencies/dependencies.ts index 9cbcdf86df..cde4fe256c 100644 --- a/cli/src/commands/dependencies/dependencies.ts +++ b/cli/src/commands/dependencies/dependencies.ts @@ -43,70 +43,66 @@ export async function pushWorkspaceDependencies( _befObj: any, newDependenciesContent: string, ): Promise { - try { - const res = workspaceDependenciesPathToLanguageAndFilename(path); - if (!res) { - throw new Error(`Unknown workspace dependencies file format: ${path}`); - } - - const { language, name } = res; - - const displayName = name - ? `named dependencies "${name}"` - : `workspace default dependencies`; - - // Fetch remote workspace dependencies and compare content directly - try { - const remoteDeps = await wmill.getLatestWorkspaceDependencies({ - workspace, - language, - name, - }); - - if (remoteDeps && remoteDeps.content === newDependenciesContent) { - log.info( - colors.green( - `${displayName} for ${language} are up-to-date, skipping push`, - ), - ); - return; - } - } catch (e: any) { - // If 404 or not found, the dependency doesn't exist remotely yet - proceed with push - if (e.status !== 404 && !e.message?.includes("not found")) { - throw e; - } - } - - log.info( - colors.yellow( - `Pushing ${ - name ? "named" : "workspace default" - } dependencies for ${language}...`, - ), + const res = workspaceDependenciesPathToLanguageAndFilename(path); + if (!res) { + throw new Error( + `Unknown workspace dependencies file format: ${path}. ` + + `Valid files: package.json, requirements.in, composer.json, go.mod, modules.json` ); + } - await wmill.createWorkspaceDependencies({ + const { language, name } = res; + + const displayName = name + ? `named dependencies "${name}"` + : `workspace default dependencies`; + + // Fetch remote workspace dependencies and compare content directly + try { + const remoteDeps = await wmill.getLatestWorkspaceDependencies({ workspace, - requestBody: { - name, - content: newDependenciesContent, - language, - workspace_id: workspace, - // Description is not supported in cli, it will use old description - description: undefined, - }, + language, + name, }); - log.info( - colors.green(`Successfully pushed ${displayName} for ${language}`), - ); - } catch (error: any) { - log.error( - colors.red(`Failed to push workspace dependencies: ${error.message}`), - ); - throw error; + if (remoteDeps && remoteDeps.content === newDependenciesContent) { + log.info( + colors.green( + `${displayName} for ${language} are up-to-date, skipping push`, + ), + ); + return; + } + } catch (e: any) { + // If 404 or not found, the dependency doesn't exist remotely yet - proceed with push + if (e.status !== 404 && !e.message?.includes("not found")) { + throw e; + } } + + log.info( + colors.yellow( + `Pushing ${ + name ? "named" : "workspace default" + } dependencies for ${language}...`, + ), + ); + + await wmill.createWorkspaceDependencies({ + workspace, + requestBody: { + name, + content: newDependenciesContent, + language, + workspace_id: workspace, + // Description is not supported in cli, it will use old description + description: undefined, + }, + }); + + log.info( + colors.green(`Successfully pushed ${displayName} for ${language}`), + ); } export default command; diff --git a/cli/src/commands/dev/dev.ts b/cli/src/commands/dev/dev.ts index 41b6b8fd9c..5352270f15 100644 --- a/cli/src/commands/dev/dev.ts +++ b/cli/src/commands/dev/dev.ts @@ -236,7 +236,7 @@ async function dev(opts: GlobalOptions & SyncOptions) { } const command = new Command() - .description("Launch a dev server that will spawn a webserver with HMR") + .description("Launch a dev server that watches for local file changes and auto-pushes them to the remote workspace. Provides live reload for scripts and flows during development.") .option( "--includes ", "Filter paths givena glob pattern or path" diff --git a/cli/src/commands/docs/docs.ts b/cli/src/commands/docs/docs.ts index 288faf335d..d86d4ca367 100644 --- a/cli/src/commands/docs/docs.ts +++ b/cli/src/commands/docs/docs.ts @@ -106,7 +106,7 @@ async function docs( const command = new Command() .name("docs") - .description("Search Windmill documentation. Requires Enterprise Edition.") + .description("Search Windmill documentation.") .arguments("") .option("--json", "Output results as JSON.") .action(docs as any); diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 58d0777f90..2ddaaa36c7 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -7,9 +7,11 @@ import * as log from "../../core/log.ts"; import { sep as SEP } from "node:path"; import { stringify as yamlStringify } from "yaml"; import { yamlParseFile } from "../../utils/yaml.ts"; +import { validateRequiredArgs } from "../../utils/utils.ts"; import * as wmill from "../../../gen/services.gen.ts"; import { readFile } from "node:fs/promises"; import { mkdirSync, writeFileSync } from "node:fs"; +import { buildFolderPath, getMetadataFileName, loadNonDottedPathsSetting } from "../../utils/resource_folders.ts"; import { requireLogin } from "../../core/auth.ts"; import { resolveWorkspace, validatePath } from "../../core/context.ts"; @@ -152,18 +154,27 @@ export async function pushFlow( const localFlow = (await yamlParseFile(localPath + "flow.yaml")) as FlowFile; const fileReader = async (path: string) => await readFile(localPath + path, "utf-8"); + const missingFiles: string[] = []; await replaceInlineScripts( localFlow.value.modules, fileReader, log, localPath, - SEP + SEP, + undefined, + missingFiles ); if (localFlow.value.failure_module) { - await replaceInlineScripts([localFlow.value.failure_module], fileReader, log, localPath, SEP); + await replaceInlineScripts([localFlow.value.failure_module], fileReader, log, localPath, SEP, undefined, missingFiles); } if (localFlow.value.preprocessor_module) { - await replaceInlineScripts([localFlow.value.preprocessor_module], fileReader, log, localPath, SEP); + await replaceInlineScripts([localFlow.value.preprocessor_module], fileReader, log, localPath, SEP, undefined, missingFiles); + } + if (missingFiles.length > 0) { + log.warn(colors.yellow( + `Warning: missing inline script file(s): ${missingFiles.join(", ")}. ` + + `The flow will be pushed with unresolved !inline references.` + )); } if (flow) { @@ -203,20 +214,21 @@ export async function pushFlow( type Options = GlobalOptions; -async function push(opts: Options, filePath: string, remotePath: string) { +async function push(opts: Options & { message?: string }, filePath: string, remotePath: string) { if (!validatePath(remotePath)) { return; } const workspace = await resolveWorkspace(opts); await requireLogin(opts); - await pushFlow(workspace.workspaceId, remotePath, filePath); + await pushFlow(workspace.workspaceId, remotePath, filePath, opts.message); log.info(colors.bold.underline.green("Flow pushed")); } async function list( opts: GlobalOptions & { showArchived?: boolean; includeDraftOnly?: boolean; json?: boolean } ) { + if (opts.json) log.setSilent(true); const workspace = await resolveWorkspace(opts); await requireLogin(opts); @@ -250,6 +262,7 @@ async function list( } } async function get(opts: GlobalOptions & { json?: boolean }, path: string) { + if (opts.json) log.setSilent(true); const workspace = await resolveWorkspace(opts); await requireLogin(opts); const f = await wmill.getFlowByPath({ @@ -264,6 +277,31 @@ async function get(opts: GlobalOptions & { json?: boolean }, path: string) { console.log(colors.bold("Description:") + " " + (f.description ?? "")); console.log(colors.bold("Edited by:") + " " + (f.edited_by ?? "")); console.log(colors.bold("Edited at:") + " " + (f.edited_at ?? "")); + // API response type doesn't include flow value/modules — cast needed to access them + const modules = (f as any).value?.modules; + if (modules && Array.isArray(modules) && modules.length > 0) { + console.log(colors.bold("Steps:")); + function printModules(mods: any[], indent: string = " ") { + for (const mod of mods) { + const type = mod.value?.type ?? "unknown"; + const detail = mod.value?.language ?? mod.value?.path ?? ""; + console.log(`${indent}${mod.id}: ${type}${detail ? " (" + detail + ")" : ""}`); + if (type === "branchall" || type === "branchone") { + for (const branch of mod.value?.branches ?? []) { + console.log(`${indent} Branch: ${branch.summary || "(default)"}`); + if (branch.modules) printModules(branch.modules, indent + " "); + } + if (type === "branchone" && mod.value?.default) { + console.log(`${indent} Default:`); + printModules(mod.value.default, indent + " "); + } + } else if (type === "forloopflow" || type === "whileloopflow") { + if (mod.value?.modules) printModules(mod.value.modules, indent + " "); + } + } + } + printModules(modules); + } } } @@ -274,54 +312,177 @@ async function run( }, path: string ) { + if (opts.silent) { + log.setSilent(true); + } const workspace = await resolveWorkspace(opts); await requireLogin(opts); const input = opts.data ? await resolve(opts.data) : {}; + // Validate required args against schema when no data provided + if (!opts.data) { + try { + const flow = await wmill.getFlowByPath({ + workspace: workspace.workspaceId, + path, + }); + validateRequiredArgs(flow.schema as Record); + } catch (e: any) { + if (e.message?.startsWith("Missing required")) throw e; + log.warn(`Could not fetch schema to validate args: ${e.message}`); + } + } + const id = await wmill.runFlowByPath({ workspace: workspace.workspaceId, path, requestBody: input, }); + // Build step label map from raw_flow if available + const stepLabels = new Map(); + try { + const initialJob = await wmill.getJob({ + workspace: workspace.workspaceId, + id, + }); + const rawFlow = (initialJob as any).raw_flow; + if (rawFlow?.modules) { + for (const mod of rawFlow.modules) { + if (mod.id) { + const label = mod.summary ? `${mod.id}: ${mod.summary}` : mod.id; + stepLabels.set(mod.id, label); + } + } + } + } catch { + // Best-effort — fall back to module IDs + } + let i = 0; + let lastStatus = ""; while (true) { const jobInfo = await wmill.getJob({ workspace: workspace.workspaceId, id, }); - if (jobInfo.flow_status!.modules.length <= i) { + + // Check if flow has completed (success or failure) + const isCompleted = (jobInfo as any).type === "CompletedJob"; + const flowStatus = jobInfo.flow_status!; + + if (flowStatus.modules.length <= i) { break; } - const module = jobInfo.flow_status!.modules[i]; + const module = flowStatus.modules[i]; - if (module.job) { - if (!opts.silent) { - log.info("====== Job " + (i + 1) + " ======"); + // If a module has failed, track its job (to show error logs), then break + if (module.type === "Failure") { + if (module.job && !opts.silent) { + const label = stepLabels.get(module.id!) ?? `Step ${i + 1}`; + log.info("====== " + label + " ======"); await track_job(workspace.workspaceId, module.job); } + break; + } + + if (module.job) { + const label = stepLabels.get(module.id!) ?? `Step ${i + 1}`; + const isForLoop = (module as any).flow_jobs !== undefined; + + if (isForLoop) { + // For-loop: track iterations as they appear, re-polling until module completes + let trackedIterations = 0; + let forLoopFailed = false; + while (true) { + const refreshed = await wmill.getJob({ + workspace: workspace.workspaceId, + id, + }); + const refreshedModule = refreshed.flow_status!.modules[i]; + const flowJobs = ((refreshedModule as any).flow_jobs as string[] | undefined) ?? []; + + // Track any new iterations + while (trackedIterations < flowJobs.length) { + if (!opts.silent) { + log.info(`====== ${label} (iteration ${trackedIterations}) ======`); + await track_job(workspace.workspaceId, flowJobs[trackedIterations]); + } + trackedIterations++; + } + + if (refreshedModule.type === "Success" || refreshedModule.type === "Failure") { + forLoopFailed = refreshedModule.type === "Failure"; + break; + } + await new Promise((resolve) => setTimeout(resolve, 200)); + } + if (forLoopFailed) break; + } else { + if (!opts.silent) { + log.info("====== " + label + " ======"); + await track_job(workspace.workspaceId, module.job); + } + } } else { - if (!opts.silent) { - log.info(module.type); + // Module not started yet — deduplicate status messages + const status = String(module.type); + if (!opts.silent && status !== lastStatus) { + log.info(colors.dim(status)); + lastStatus = status; } await new Promise((resolve, _) => setTimeout(() => resolve(undefined), 100) ); + + // If flow already completed while we were waiting, break out + if (isCompleted) break; + continue; } + lastStatus = ""; i++; } - if (!opts.silent) { - log.info(colors.green.underline.bold("Flow ran to completion")); - log.info("\n"); + // Wait for flow completion with retry (handles race when --silent skips module tracking) + const MAX_RETRIES = 600; // ~60 seconds at 100ms intervals + let retries = 0; + while (retries < MAX_RETRIES) { + try { + const jobInfo = await wmill.getCompletedJob({ + workspace: workspace.workspaceId, + id, + }); + + if (!opts.silent) { + if (jobInfo.success === false) { + log.info(colors.red.underline.bold("Flow failed")); + } else { + log.info(colors.green.underline.bold("Flow ran to completion")); + } + log.info("\n"); + } + + if (jobInfo.success === false) { + process.exitCode = 1; + } + + if (opts.silent) { + console.log(JSON.stringify(jobInfo.result ?? {})); + } else { + log.info(JSON.stringify(jobInfo.result ?? {}, null, 2)); + } + + break; + } catch { + retries++; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + if (retries >= MAX_RETRIES) { + throw new Error(`Timed out waiting for flow ${id} to complete`); } - const jobInfo = await wmill.getCompletedJob({ - workspace: workspace.workspaceId, - id, - }); - log.info(JSON.stringify(jobInfo.result ?? {}, null, 2)); } async function preview( @@ -332,6 +493,9 @@ async function preview( } & SyncOptions, flowPath: string ) { + if (opts.silent) { + log.setSilent(true); + } const useLocalPathScripts = !opts.remote; if (useLocalPathScripts) { opts = await mergeConfigWithConfigFile(opts); @@ -340,14 +504,16 @@ async function preview( await requireLogin(opts); const codebases = useLocalPathScripts ? listSyncCodebases(opts) : []; - // Normalize path - ensure it's a directory path to a .flow folder - if (!flowPath.endsWith(".flow") && !flowPath.endsWith(".flow" + SEP)) { + // Normalize path - ensure it's a directory path to a .flow or __flow folder + const isFlowDir = flowPath.endsWith(".flow") || flowPath.endsWith(".flow" + SEP) + || flowPath.endsWith("__flow") || flowPath.endsWith("__flow" + SEP); + if (!isFlowDir) { // Check if it's a flow.yaml file if (flowPath.endsWith("flow.yaml") || flowPath.endsWith("flow.json")) { flowPath = flowPath.substring(0, flowPath.lastIndexOf(SEP)); } else { throw new Error( - "Flow path must be a .flow directory or a flow.yaml file" + "Flow path must be a .flow/__flow directory or a flow.yaml file" ); } } @@ -421,13 +587,23 @@ async function preview( }); } catch (e: any) { if (e.body) { - log.error(`Flow preview failed: ${JSON.stringify(e.body)}`); + // If a failure_module ran, the body contains its result — not an error + if (e.body.result !== undefined) { + if (opts.silent) { + console.log(JSON.stringify(e.body.result)); + } else { + log.info(colors.yellow.bold("Flow failed, error handler result:")); + log.info(JSON.stringify(e.body.result, null, 2)); + } + process.exitCode = 1; + return; + } } throw e; } if (opts.silent) { - console.log(JSON.stringify(result, null, 2)); + console.log(JSON.stringify(result)); } else { log.info(colors.bold.underline.green("Flow preview completed")); log.info(JSON.stringify(result, null, 2)); @@ -516,7 +692,7 @@ export async function generateLocks( } } -export function bootstrap( +export async function bootstrap( opts: GlobalOptions & { summary: string; description: string }, flowPath: string ) { @@ -524,8 +700,10 @@ export function bootstrap( return; } - const flowDirFullPath = `${flowPath}.flow`; - mkdirSync(flowDirFullPath, { recursive: false }); + await loadNonDottedPathsSetting(); + + const flowDirFullPath = buildFolderPath(flowPath, "flow"); + mkdirSync(flowDirFullPath, { recursive: true }); const newFlowDefinition = defaultFlowDefinition(); if (opts.summary !== undefined) { @@ -539,10 +717,76 @@ export function bootstrap( newFlowDefinition as Record ); - const flowYamlPath = `${flowDirFullPath}/flow.yaml`; + const metadataFile = getMetadataFileName("flow", "yaml"); + const flowYamlPath = `${flowDirFullPath}/${metadataFile}`; writeFileSync(flowYamlPath, newFlowDefinitionYaml, { flag: "wx", encoding: "utf-8" }); } +async function history( + opts: GlobalOptions & { json?: boolean }, + flowPath: string +) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const versions = await wmill.getFlowHistory({ + workspace: workspace.workspaceId, + path: flowPath, + }); + + if (opts.json) { + console.log(JSON.stringify(versions)); + } else { + if (versions.length === 0) { + log.info("No version history found for " + flowPath); + return; + } + new Table() + .header(["Version", "Created At", "Deployment Message"]) + .padding(2) + .border(true) + .body( + versions.map((v) => [ + String(v.id), + new Date(v.created_at).toISOString().replace("T", " ").substring(0, 19), + v.deployment_msg ?? "-", + ]) + ) + .render(); + } +} + +async function showVersion( + opts: GlobalOptions & { json?: boolean }, + flowPath: string, + version: string +) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const flow = await wmill.getFlowVersion({ + workspace: workspace.workspaceId, + path: flowPath, + version: parseInt(version, 10), + }); + + if (opts.json) { + console.log(JSON.stringify(flow)); + } else { + console.log(colors.bold("Path:") + " " + flow.path); + console.log(colors.bold("Summary:") + " " + (flow.summary ?? "-")); + console.log(colors.bold("Description:") + " " + (flow.description ?? "-")); + console.log(colors.bold("Schema:")); + console.log(JSON.stringify(flow.schema, null, 2)); + console.log(colors.bold("Value:")); + console.log(JSON.stringify(flow.value, null, 2)); + } +} + const command = new Command() .description("flow related commands") .option("--show-archived", "Enable archived flows in output") @@ -561,6 +805,7 @@ const command = new Command() "push a local flow spec. This overrides any remote versions." ) .arguments(" ") + .option("--message ", "Deployment message") .action(push as any) .command("run", "run a flow by path.") .arguments("") @@ -616,6 +861,14 @@ const command = new Command() .arguments("") .option("--summary ", "flow summary") .option("--description ", "flow description") - .action(bootstrap as any); + .action(bootstrap as any) + .command("history", "Show version history for a flow") + .arguments("") + .option("--json", "Output as JSON (for piping to jq)") + .action(history as any) + .command("show-version", "Show a specific version of a flow") + .arguments(" ") + .option("--json", "Output as JSON (for piping to jq)") + .action(showVersion as any); export default command; diff --git a/cli/src/commands/flow/flow_metadata.ts b/cli/src/commands/flow/flow_metadata.ts index 1f5d86b8ea..57a9289fa1 100644 --- a/cli/src/commands/flow/flow_metadata.ts +++ b/cli/src/commands/flow/flow_metadata.ts @@ -18,12 +18,12 @@ import { filterWorkspaceDependenciesForScripts, } from "../../utils/metadata.ts"; import { ScriptLanguage } from "../../utils/script_common.ts"; -import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; +import { extractInlineScripts as extractInlineScriptsForFlows, extractCurrentMapping } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; import { newPathAssigner } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts"; import { exts } from "../script/script.ts"; -import { FSFSElement } from "../sync/sync.ts"; +import { FSFSElement, yamlOptions } from "../sync/sync.ts"; import { Workspace } from "../workspace/workspace.ts"; import { FlowFile } from "./flow.ts"; import { FlowValue } from "../../../gen/types.gen.ts"; @@ -188,6 +188,17 @@ export async function generateFlowLockInternal( log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`); } const fileReader = async (path: string) => await readFile(folder + SEP + path, "utf-8"); + + // Capture existing module-ID-to-file-path mapping before replaceInlineScripts + // overwrites the !inline references with actual file content. This preserves + // the original filenames when re-extracting inline scripts after lock generation. + const currentMapping = extractCurrentMapping( + flowValue.value.modules, + {}, + flowValue.value.failure_module, + flowValue.value.preprocessor_module, + ); + // In tree mode, use the tree's staleness info (which includes transitive dependency changes) // to determine which scripts need relocking, instead of only content-changed ones. const locksToRemove = (tree && !legacyBehaviour) @@ -215,6 +226,12 @@ export async function generateFlowLockInternal( //removeChangedLocks const tempScriptRefs = tree?.getTempScriptRefs(folderNormalized); + + // Preserve notes and groups — the backend round-trips through FlowValue + // which doesn't include these fields, so they'd be lost (#8641). + const savedNotes = flowValue.value.notes; + const savedGroups = flowValue.value.groups; + flowValue.value = await updateFlow( workspace, flowValue.value, @@ -223,21 +240,25 @@ export async function generateFlowLockInternal( tempScriptRefs ); + // Restore notes and groups that the backend stripped + if (savedNotes !== undefined) flowValue.value.notes = savedNotes; + if (savedGroups !== undefined) flowValue.value.groups = savedGroups; + const lockAssigner = newPathAssigner(opts.defaultTs ?? "bun", { skipInlineScriptSuffix: getNonDottedPaths(), }); const inlineScripts = extractInlineScriptsForFlows( flowValue.value.modules, - {}, + currentMapping, SEP, opts.defaultTs, lockAssigner ); if (flowValue.value.failure_module) { - inlineScripts.push(...extractInlineScriptsForFlows([flowValue.value.failure_module], {}, SEP, opts.defaultTs, lockAssigner)); + inlineScripts.push(...extractInlineScriptsForFlows([flowValue.value.failure_module], currentMapping, SEP, opts.defaultTs, lockAssigner)); } if (flowValue.value.preprocessor_module) { - inlineScripts.push(...extractInlineScriptsForFlows([flowValue.value.preprocessor_module], {}, SEP, opts.defaultTs, lockAssigner)); + inlineScripts.push(...extractInlineScriptsForFlows([flowValue.value.preprocessor_module], currentMapping, SEP, opts.defaultTs, lockAssigner)); } inlineScripts.forEach((s) => { writeIfChanged(process.cwd() + SEP + folder + SEP + s.path, s.content); @@ -246,7 +267,7 @@ export async function generateFlowLockInternal( // Overwrite `flow.yaml` with the new lockfile references writeIfChanged( process.cwd() + SEP + folder + SEP + "flow.yaml", - yamlStringify(flowValue as Record) + yamlStringify(flowValue as Record, yamlOptions) ); } diff --git a/cli/src/commands/folder/folder.ts b/cli/src/commands/folder/folder.ts index eb5a51c4f8..bd298e6f88 100644 --- a/cli/src/commands/folder/folder.ts +++ b/cli/src/commands/folder/folder.ts @@ -22,6 +22,7 @@ export interface FolderFile { } async function list(opts: GlobalOptions & { json?: boolean }) { + if (opts.json) log.setSilent(true); const workspace = await resolveWorkspace(opts); await requireLogin(opts); diff --git a/cli/src/commands/generate-metadata/generate-metadata.ts b/cli/src/commands/generate-metadata/generate-metadata.ts index d273f4b631..7abd155301 100644 --- a/cli/src/commands/generate-metadata/generate-metadata.ts +++ b/cli/src/commands/generate-metadata/generate-metadata.ts @@ -355,71 +355,102 @@ async function generateMetadata( return colors.dim(colors.white(`[${n}/${total}]`.padEnd(maxWidth, " "))); }; + const errors: { path: string; error: string }[] = []; + // Process scripts for (const item of scripts) { current++; log.info(`${formatProgress(current)} script ${item.path}`); - await generateScriptMetadataInternal( - item.path, // originalPath with extension - workspace, - opts, - false, // dryRun - true, // noStaleMessage - mismatchedWorkspaceDeps, - codebases, - false, - false, // legacyBehaviour - tree - ); + try { + await generateScriptMetadataInternal( + item.path, // originalPath with extension + workspace, + opts, + false, // dryRun + true, // noStaleMessage + mismatchedWorkspaceDeps, + codebases, + false, + false, // legacyBehaviour + tree + ); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + errors.push({ path: item.path, error: msg }); + log.error(` Failed: ${msg}`); + } } // Process flows for (const item of flows) { current++; - const result = await generateFlowLockInternal( - item.folder.replaceAll("/", SEP), - false, // dryRun - workspace, - opts, - false, - true, // noStaleMessage - false, // legacyBehaviour - tree - ); - const flowResult = result as FlowLocksResult | undefined; - const scriptsInfo = flowResult?.updatedScripts?.length - ? colors.dim(colors.white(`: ${flowResult.updatedScripts.join(", ")}`)) - : ""; - log.info(`${formatProgress(current)} flow ${item.path}${scriptsInfo}`); + try { + const result = await generateFlowLockInternal( + item.folder.replaceAll("/", SEP), + false, // dryRun + workspace, + opts, + false, + true, // noStaleMessage + false, // legacyBehaviour + tree + ); + const flowResult = result as FlowLocksResult | undefined; + const scriptsInfo = flowResult?.updatedScripts?.length + ? colors.dim(colors.white(`: ${flowResult.updatedScripts.join(", ")}`)) + : ""; + log.info(`${formatProgress(current)} flow ${item.path}${scriptsInfo}`); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + errors.push({ path: item.path, error: msg }); + log.info(`${formatProgress(current)} flow ${item.path}`); + log.error(` Failed: ${msg}`); + } } // Process apps for (const item of apps) { current++; - const result = await generateAppLocksInternal( - item.folder.replaceAll("/", SEP), - item.isRawApp!, // rawApp - false, // dryRun - workspace, - opts, - false, - true, // noStaleMessage - false, // legacyBehaviour - tree - ); - const appResult = result as AppLocksResult | undefined; - const scriptsInfo = appResult?.updatedScripts?.length - ? colors.dim(colors.white(`: ${appResult.updatedScripts.join(", ")}`)) - : ""; - log.info(`${formatProgress(current)} app ${item.path}${scriptsInfo}`); + try { + const result = await generateAppLocksInternal( + item.folder.replaceAll("/", SEP), + item.isRawApp!, // rawApp + false, // dryRun + workspace, + opts, + false, + true, // noStaleMessage + false, // legacyBehaviour + tree + ); + const appResult = result as AppLocksResult | undefined; + const scriptsInfo = appResult?.updatedScripts?.length + ? colors.dim(colors.white(`: ${appResult.updatedScripts.join(", ")}`)) + : ""; + log.info(`${formatProgress(current)} app ${item.path}${scriptsInfo}`); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + errors.push({ path: item.path, error: msg }); + log.info(`${formatProgress(current)} app ${item.path}`); + log.error(` Failed: ${msg}`); + } } // Persist all stale workspace dep hashes (not just filtered — deps are global, not folder-scoped) const allStaleDeps = staleItems.filter((i) => i.type === "dependencies"); await tree.persistDepsHashes(allStaleDeps.map((d) => d.path)); + const succeeded = total - errors.length; log.info(""); - log.info(`Done. Updated ${colors.bold(String(total))} item(s).`); + if (errors.length > 0) { + log.info(`Done. Updated ${colors.bold(String(succeeded))}/${total} item(s). ${colors.red(String(errors.length) + " failed")}:`); + for (const { path, error } of errors) { + log.error(` ${path}: ${error}`); + } + process.exitCode = 1; + } else { + log.info(`Done. Updated ${colors.bold(String(total))} item(s).`); + } } const command = new Command() diff --git a/cli/src/commands/group/group.ts b/cli/src/commands/group/group.ts new file mode 100644 index 0000000000..5d06a7df1f --- /dev/null +++ b/cli/src/commands/group/group.ts @@ -0,0 +1,157 @@ +import { GlobalOptions } from "../../types.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { Command } from "@cliffy/command"; +import { Table } from "@cliffy/table"; +import { colors } from "@cliffy/ansi/colors"; +import * as log from "../../core/log.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import * as wmill from "../../../gen/services.gen.ts"; + +async function list(opts: GlobalOptions & { json?: boolean }) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const groups = await wmill.listGroups({ + workspace: workspace.workspaceId, + }); + + if (opts.json) { + console.log(JSON.stringify(groups)); + } else { + if (groups.length === 0) { + log.info("No groups found."); + return; + } + new Table() + .header(["Name", "Summary", "Members"]) + .padding(2) + .border(true) + .body( + groups.map((g) => [ + g.name, + g.summary ?? "-", + String(g.members?.length ?? 0), + ]) + ) + .render(); + } +} + +async function get( + opts: GlobalOptions & { json?: boolean }, + name: string +) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const group = await wmill.getGroup({ + workspace: workspace.workspaceId, + name, + }); + + if (opts.json) { + console.log(JSON.stringify(group)); + } else { + console.log(colors.bold("Name:") + " " + group.name); + console.log(colors.bold("Summary:") + " " + (group.summary ?? "-")); + console.log( + colors.bold("Members:") + + " " + + (group.members && group.members.length > 0 + ? group.members.join(", ") + : "(none)") + ); + } +} + +async function create( + opts: GlobalOptions & { summary?: string }, + name: string +) { + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + await wmill.createGroup({ + workspace: workspace.workspaceId, + requestBody: { + name, + summary: opts.summary, + }, + }); + + log.info(colors.green(`Group '${name}' created.`)); +} + +async function deleteGroup(opts: GlobalOptions, name: string) { + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + await wmill.deleteGroup({ + workspace: workspace.workspaceId, + name, + }); + + log.info(colors.green(`Group '${name}' deleted.`)); +} + +async function addUser(opts: GlobalOptions, name: string, username: string) { + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + await wmill.addUserToGroup({ + workspace: workspace.workspaceId, + name, + requestBody: { username }, + }); + + log.info(colors.green(`User '${username}' added to group '${name}'.`)); +} + +async function removeUser(opts: GlobalOptions, name: string, username: string) { + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + await wmill.removeUserToGroup({ + workspace: workspace.workspaceId, + name, + requestBody: { username }, + }); + + log.info(colors.green(`User '${username}' removed from group '${name}'.`)); +} + +const command = new Command() + .description("Manage workspace groups") + .option("--json", "Output as JSON (for piping to jq)") + .action(list as any) + .command("list", "List all groups in the workspace") + .option("--json", "Output as JSON (for piping to jq)") + .action(list as any) + .command("get", "Get group details and members") + .arguments("") + .option("--json", "Output as JSON (for piping to jq)") + .action(get as any) + .command("create", "Create a new group") + .arguments("") + .option("--summary ", "Group summary/description") + .action(create as any) + .command("delete", "Delete a group") + .arguments("") + .action(deleteGroup as any) + .command("add-user", "Add a user to a group") + .arguments(" ") + .action(addUser as any) + .command("remove-user", "Remove a user from a group") + .arguments(" ") + .action(removeUser as any); + +export default command; diff --git a/cli/src/commands/init/init.ts b/cli/src/commands/init/init.ts index 5883967b77..db4950575b 100644 --- a/cli/src/commands/init/init.ts +++ b/cli/src/commands/init/init.ts @@ -3,13 +3,14 @@ import { colors } from "@cliffy/ansi/colors"; import { Command } from "@cliffy/command"; import { Confirm } from "@cliffy/prompt/confirm"; import * as log from "../../core/log.ts"; -import { stringify as yamlStringify } from "yaml"; +import { type BranchBinding } from "./template.ts"; import { GlobalOptions } from "../../types.ts"; import { readLockfile } from "../../utils/metadata.ts"; import { getActiveWorkspaceOrFallback } from "../workspace/workspace.ts"; import { generateRTNamespace } from "../resource-type/resource-type.ts"; import { SKILLS, SKILL_CONTENT, SCHEMAS, SCHEMA_MAPPINGS } from "../../guidance/skills.ts"; import { generateAgentsMdContent } from "../../guidance/core.ts"; +import { generateCommentedTemplate } from "./template.ts"; /** * Format a YAML schema for inclusion in skill markdown files. @@ -42,61 +43,37 @@ export interface InitOptions { */ async function initAction(opts: InitOptions) { if (await stat("wmill.yaml").catch(() => null)) { - log.error(colors.red("wmill.yaml already exists")); + log.info("wmill.yaml already exists, skipping config generation"); } else { - // Import DEFAULT_SYNC_OPTIONS from conf.ts - const { DEFAULT_SYNC_OPTIONS } = await import("../../core/conf.ts"); - - // Create initial config with defaults - const initialConfig = { ...DEFAULT_SYNC_OPTIONS } as any; - - // Add branch structure + // Detect current git branch for template const { isGitRepository, getCurrentGitBranch } = await import( "../../utils/git.ts" ); + let branchName: string | undefined; + let binding: BranchBinding | undefined; if (isGitRepository()) { - const currentBranch = getCurrentGitBranch(); - if (currentBranch) { - initialConfig.gitBranches = { - [currentBranch]: { overrides: {} }, - }; - } else { - initialConfig.gitBranches = {}; - } - } else { - initialConfig.gitBranches = {}; + branchName = getCurrentGitBranch() ?? undefined; } - initialConfig.nonDottedPaths = true; - await writeFile("wmill.yaml", yamlStringify(initialConfig), "utf-8"); - log.info(colors.green("wmill.yaml created with default settings")); - - // Create lock file - await readLockfile(); - - // Offer to bind workspace profile to current branch - if (isGitRepository()) { + // Determine workspace binding before writing the template + if (isGitRepository() && branchName) { const activeWorkspace = await getActiveWorkspaceOrFallback( opts as GlobalOptions ); - const currentBranch = getCurrentGitBranch(); - if (activeWorkspace && currentBranch) { - // Determine binding behavior based on flags + if (activeWorkspace) { const shouldBind = opts.bindProfile === true; const shouldPrompt = opts.bindProfile === undefined && !!process.stdin.isTTY && !opts.useDefault; - const shouldSkip = opts.bindProfile != true && - (opts.useDefault || !!!process.stdin.isTTY); + (opts.useDefault || !process.stdin.isTTY); if (!shouldSkip) { - // Show workspace info if we're binding or prompting if (shouldBind || shouldPrompt) { log.info( - colors.yellow(`\nCurrent Git branch: ${colors.bold(currentBranch)}`) + colors.yellow(`\nCurrent Git branch: ${colors.bold(branchName)}`) ); log.info( colors.yellow( @@ -118,37 +95,31 @@ async function initAction(opts: InitOptions) { default: true, }))) ) { - // Update the config with workspace binding - const currentConfig = await import("../../core/conf.ts").then((m) => - m.readConfigFile() - ); - if (!currentConfig.gitBranches) { - currentConfig.gitBranches = {}; - } - if (!currentConfig.gitBranches[currentBranch]) { - currentConfig.gitBranches[currentBranch] = { overrides: {} }; - } - log.info( - `binding branch ${currentBranch} to workspace ${activeWorkspace.name} on ${activeWorkspace.remote}` - ); - currentConfig.gitBranches[currentBranch].baseUrl = - activeWorkspace.remote; - currentConfig.gitBranches[currentBranch].workspaceId = - activeWorkspace.workspaceId; - - await writeFile("wmill.yaml", yamlStringify(currentConfig), "utf-8"); - - log.info( - colors.green( - `✓ Bound branch '${currentBranch}' to workspace '${activeWorkspace.name}'` - ) + `binding branch ${branchName} to workspace ${activeWorkspace.name} on ${activeWorkspace.remote}` ); + binding = { + baseUrl: activeWorkspace.remote, + workspaceId: activeWorkspace.workspaceId, + }; } } } } + await writeFile("wmill.yaml", generateCommentedTemplate(branchName, binding), "utf-8"); + log.info(colors.green("wmill.yaml created with default settings")); + if (binding) { + log.info( + colors.green( + `✓ Bound branch '${branchName}' to workspace` + ) + ); + } + + // Create lock file + await readLockfile(); + // Check for backend git-sync settings unless --use-default is specified if (!opts.useDefault) { try { diff --git a/cli/src/commands/init/template.ts b/cli/src/commands/init/template.ts new file mode 100644 index 0000000000..0684b7ca14 --- /dev/null +++ b/cli/src/commands/init/template.ts @@ -0,0 +1,395 @@ +/** + * Configuration option descriptor — each entry IS a JSON Schema property + * with extra metadata for template rendering and reference table display. + * + * To generate the JSON Schema: iterate entries, strip NON_SCHEMA_KEYS, done. + * Sub-fields of complex types (codebases items, gitBranches branch config) + * are defined inline in the parent's schema — no duplicate entries needed. + * The reference table auto-expands nested schemas into rows. + * + * Adding a new option: + * 1. Add an entry to CONFIG_REFERENCE with JSON Schema type fields + description + * 2. Add template rendering hints (section, commented, templateValue, etc.) + * 3. `wmill init` (YAML template), `wmill config` (table), and wmill.schema.json all update automatically + */ +export interface ConfigOption { + // --- JSON Schema fields (kept when generating schema) --- + type: string; + description: string; + enum?: string[]; + items?: Record; + properties?: Record; + additionalProperties?: Record | boolean; + required?: string[]; + + // --- Non-schema metadata (stripped when generating schema) --- + name: string; + default: string; + + // --- Template rendering hints (also stripped) --- + section?: string; + sectionNote?: string; + commented?: boolean; + templateValue?: string; + example?: string; + inlineComment?: string; + groupNote?: string; +} + +/** Keys to strip from ConfigOption entries when generating JSON Schema. */ +const NON_SCHEMA_KEYS = new Set([ + "name", "default", + "section", "sectionNote", "commented", "templateValue", + "example", "inlineComment", "groupNote", +]); + +// Reusable sub-schemas for nested types +const SPECIFIC_ITEMS_SCHEMA = { + type: "object", + description: "Sync only specific items", + properties: { + variables: { type: "array", items: { type: "string" }, description: "Specific variable paths to sync" }, + resources: { type: "array", items: { type: "string" }, description: "Specific resource paths to sync" }, + triggers: { type: "array", items: { type: "string" }, description: "Specific trigger paths to sync" }, + folders: { type: "array", items: { type: "string" }, description: "Specific folder paths to sync" }, + settings: { type: "boolean", description: "Whether to sync settings" }, + }, + additionalProperties: false, +} as const; + +const BRANCH_CONFIG_SCHEMA = { + type: "object", + properties: { + baseUrl: { type: "string", description: "Windmill instance URL for this branch" }, + workspaceId: { type: "string", description: "Workspace ID to sync with for this branch" }, + overrides: { type: "object", description: "Override any top-level sync option for this branch" }, + promotionOverrides: { type: "object", description: "Overrides applied when using --promotion flag" }, + specificItems: SPECIFIC_ITEMS_SCHEMA, + }, + additionalProperties: false, +} as const; + +/** + * All wmill.yaml configuration options — single source of truth. + * Each entry is a JSON Schema property with extra metadata. + */ +export const CONFIG_REFERENCE: ConfigOption[] = [ + // ── Core ────────────────────────────────────────────────────────────── + { name: "defaultTs", type: "string", enum: ["bun", "deno"], default: "bun", description: "Default TypeScript runtime for new scripts" }, + { name: "includes", type: "array", items: { type: "string" }, default: '["f/**"]', description: "Glob patterns for files to include in sync", + templateValue: '\n - "f/**"' }, + { name: "extraIncludes", type: "array", items: { type: "string" }, default: "[]", description: "Additional glob patterns merged with includes (useful in branch overrides)", + commented: true }, + { name: "excludes", type: "array", items: { type: "string" }, default: "[]", description: "Glob patterns for files to exclude from sync" }, + + // ── What to sync ────────────────────────────────────────────────────── + { name: "skipVariables", type: "boolean", default: "false", description: "Skip syncing variables", + section: "What to sync", sectionNote: '"skip" options default to false (synced), "include" options default to false (not synced)' }, + { name: "skipResources", type: "boolean", default: "false", description: "Skip syncing resources" }, + { name: "skipResourceTypes", type: "boolean", default: "false", description: "Skip syncing resource types" }, + { name: "skipSecrets", type: "boolean", default: "true", description: "Skip syncing secrets (true by default for security)", + inlineComment: "true by default — secrets are not synced for security" }, + { name: "skipScripts", type: "boolean", default: "false", description: "Skip syncing scripts" }, + { name: "skipFlows", type: "boolean", default: "false", description: "Skip syncing flows" }, + { name: "skipApps", type: "boolean", default: "false", description: "Skip syncing apps" }, + { name: "skipFolders", type: "boolean", default: "false", description: "Skip syncing folders" }, + { name: "skipWorkspaceDependencies", type: "boolean", default: "false", description: "Skip syncing workspace dependencies" }, + + { name: "includeSchedules", type: "boolean", default: "false", description: "Include schedules in sync", + commented: true, templateValue: "true", groupNote: "Uncomment to include these (excluded by default):" }, + { name: "includeTriggers", type: "boolean", default: "false", description: "Include triggers (http, websocket, kafka, etc.) in sync", + commented: true, templateValue: "true" }, + { name: "includeUsers", type: "boolean", default: "false", description: "Include workspace users in sync", + commented: true, templateValue: "true" }, + { name: "includeGroups", type: "boolean", default: "false", description: "Include workspace groups in sync", + commented: true, templateValue: "true" }, + { name: "includeSettings", type: "boolean", default: "false", description: "Include workspace settings in sync", + commented: true, templateValue: "true" }, + { name: "includeKey", type: "boolean", default: "false", description: "Include encryption key in sync", + commented: true, templateValue: "true" }, + + // ── Sync behavior ───────────────────────────────────────────────────── + { name: "parallel", type: "integer", default: "(unset)", description: "Number of parallel operations during sync", + section: "Sync behavior", commented: true, templateValue: "4" }, + { name: "locksRequired", type: "boolean", default: "false", description: "Require lock files for all scripts", + commented: true, templateValue: "true" }, + { name: "lint", type: "boolean", default: "false", description: "Run linting before push", + commented: true, templateValue: "true" }, + { name: "plainSecrets", type: "boolean", default: "false", description: "Handle secrets as plain text (not recommended)", + commented: true }, + { name: "message", type: "string", default: "(unset)", description: "Default commit message for sync operations", + commented: true, templateValue: '"my commit message"' }, + { name: "promotion", type: "string", default: "(unset)", description: "Branch name to use promotion overrides from during sync", + commented: true, templateValue: "staging" }, + { name: "skipBranchValidation", type: "boolean", default: "false", description: "Skip validation that current git branch matches a configured branch", + commented: true }, + { name: "nonDottedPaths", type: "boolean", default: "true", description: "Use __flow/__app/__raw_app suffixes instead of .flow/.app/.raw_app" }, + + // ── Codebase bundling ───────────────────────────────────────────────── + { name: "codebases", type: "array", default: "[]", description: "Codebase bundling configurations for shared libraries", + items: { + type: "object", + properties: { + relative_path: { type: "string", description: "Path to the codebase directory" }, + includes: { type: "array", items: { type: "string" }, description: "Glob patterns for files to include in bundle" }, + excludes: { type: "array", items: { type: "string" }, description: "Glob patterns for files to exclude from bundle" }, + format: { type: "string", enum: ["cjs", "esm"], description: "Bundle output format" }, + external: { type: "array", items: { type: "string" }, description: "Dependencies to leave unbundled (externals)" }, + assets: { type: "array", items: { type: "object", properties: { from: { type: "string" }, to: { type: "string" } }, required: ["from", "to"] }, description: "Static files to copy into the bundle" }, + customBundler: { type: "string", description: "Path to a custom bundler script (replaces esbuild)" }, + inject: { type: "array", items: { type: "string" }, description: "Files to inject into every entry point" }, + define: { type: "object", additionalProperties: { type: "string" }, description: "Compile-time constant definitions" }, + banner: { type: "object", additionalProperties: { type: "string" }, description: "Text to prepend to output files by type" }, + loader: { type: "object", additionalProperties: { type: "string" }, description: "esbuild loader overrides by extension" }, + }, + required: ["relative_path"], + additionalProperties: false, + }, + section: "Codebase bundling (shared libraries)", + sectionNote: "Bundle TypeScript/JavaScript codebases that scripts import from.\nEach entry is bundled and uploaded so scripts can import shared code.", + example: [ + "# codebases:", + '# - relative_path: ./shared # path to the codebase', + '# includes: ["**/*.ts"] # files to include in bundle', + '# excludes: ["node_modules/**"] # files to exclude', + '# format: esm # bundle format: "cjs" or "esm"', + '# external: ["pg", "axios"] # dependencies to leave unbundled', + "# assets: # static files to copy into bundle", + "# - from: ./static", + "# to: ./dist", + "# # customBundler: ./build.ts # custom bundler script (replaces esbuild)", + '# # inject: ["./polyfills.ts"] # files to inject into every entry point', + "# # define: # compile-time constants", + "# # API_URL: '\"https://api.example.com\"'", + "# # banner: # text prepended to output files", + '# # js: "/* bundled by windmill */"', + "# # loader: # esbuild loader overrides", + '# # ".png": "dataurl"', + ].join("\n"), + }, + + // ── Git branches ────────────────────────────────────────────────────── + { name: "gitBranches", type: "object", default: "{}", description: "Map git branches to workspaces and per-branch sync overrides", + properties: { commonSpecificItems: SPECIFIC_ITEMS_SCHEMA }, + additionalProperties: BRANCH_CONFIG_SCHEMA, + section: "Git branch / environment bindings", + sectionNote: "Map git branches to Windmill workspaces and override settings per branch.\nUse \"environments\" as an alias if you prefer environment-based terminology.", + templateValue: "\n {{BRANCH}}:\n overrides: {}", + example: [ + "{{BASEURL_LINE}}", + "{{WORKSPACE_ID_LINE}}", + " # promotionOverrides: # overrides applied during --promotion", + " # skipSecrets: false", + " # specificItems: # only sync these specific items", + ' # variables: ["f/my_folder/my_var"]', + ' # resources: ["f/my_folder/my_res"]', + ' # triggers: ["f/my_folder/my_trigger"]', + ' # folders: ["my_folder"]', + " # settings: true", + "", + " # Example: staging branch bound to a different workspace", + " # staging:", + " # baseUrl: https://staging.windmill.dev", + " # workspaceId: staging-workspace", + " # overrides:", + " # skipSecrets: false", + " # includeSchedules: true", + "", + " # Items shared across ALL branches", + " # commonSpecificItems:", + ' # variables: ["f/shared/api_key"]', + ' # resources: ["f/shared/db_conn"]', + ' # folders: ["shared"]', + ].join("\n"), + }, + + { name: "environments", type: "object", default: "{}", description: "Alias for gitBranches — use if you prefer environment-based terminology", + properties: { commonSpecificItems: SPECIFIC_ITEMS_SCHEMA }, + additionalProperties: BRANCH_CONFIG_SCHEMA, + commented: true }, +]; + +// ─── Template generator ───────────────────────────────────────────────────── + +export interface BranchBinding { + baseUrl: string; + workspaceId: string; +} + +/** Quote a string for use as a YAML key if it contains special characters. */ +function yamlKey(s: string): string { + if ( + /^[a-zA-Z0-9_/.@-]+$/.test(s) && + !/^(true|false|yes|no|on|off|null|~)$/i.test(s) && + !/^\d+(\.\d+)?$/.test(s) + ) { + return s; + } + return `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; +} + +export function generateCommentedTemplate(branchName?: string, binding?: BranchBinding): string { + const branch = yamlKey(branchName ?? "main"); + const lines: string[] = [ + "# yaml-language-server: $schema=wmill.schema.json", + "# wmill.yaml — Windmill CLI configuration", + '# Full reference: run "wmill config"', + "", + ]; + + for (const opt of CONFIG_REFERENCE) { + if (opt.section) { + const ruler = "-".repeat(Math.max(0, 65 - opt.section.length)); + lines.push(`# --- ${opt.section} ${ruler}`); + if (opt.sectionNote) { + for (const noteLine of opt.sectionNote.split("\n")) { + lines.push(`# ${noteLine}`); + } + } + lines.push(""); + } + + if (opt.groupNote) { + lines.push(`# ${opt.groupNote}`); + } + + const value = opt.templateValue ?? opt.default; + const resolvedValue = value.replace("{{BRANCH}}", branch); + + if (opt.commented) { + lines.push(`# ${opt.description}`); + lines.push(`# ${opt.name}: ${resolvedValue}`); + } else { + lines.push(`# ${opt.description}`); + if (opt.inlineComment) { + const base = `${opt.name}: ${resolvedValue}`; + const pad = " ".repeat(Math.max(1, 32 - base.length)); + lines.push(`${base}${pad}# ${opt.inlineComment}`); + } else { + lines.push(`${opt.name}: ${resolvedValue}`); + } + } + + if (opt.example) { + let resolvedExample = opt.example.replace(/\{\{BRANCH\}\}/g, branch); + if (binding) { + resolvedExample = resolvedExample + .replace("{{BASEURL_LINE}}", ` baseUrl: ${binding.baseUrl}`) + .replace("{{WORKSPACE_ID_LINE}}", ` workspaceId: ${binding.workspaceId}`); + } else { + resolvedExample = resolvedExample + .replace("{{BASEURL_LINE}}", " # baseUrl: https://app.windmill.dev # Windmill instance URL for this branch") + .replace("{{WORKSPACE_ID_LINE}}", " # workspaceId: my-workspace # workspace to sync with"); + } + for (const exLine of resolvedExample.split("\n")) { + lines.push(exLine); + } + } + + lines.push(""); + } + + return lines.join("\n"); +} + +// ─── Reference formatters ─────────────────────────────────────────────────── + +/** Recursively expand a schema's properties into flat reference rows. */ +function expandSchema( + prefix: string, + schema: Record, + rows: { name: string; description: string; default: string }[] +): void { + if (schema.properties) { + for (const [key, prop] of Object.entries(schema.properties) as [string, Record][]) { + const name = prefix ? `${prefix}.${key}` : key; + rows.push({ name, description: prop.description ?? "", default: "" }); + // Recurse into nested object properties (e.g., specificItems) + if (prop.properties && prop.type === "object") { + expandSchema(name, prop, rows); + } + } + } +} + +export function formatConfigReference(): string { + const nameWidth = 48; + const descWidth = 70; + + const header = [ + "OPTION".padEnd(nameWidth), + "DESCRIPTION".padEnd(descWidth), + "DEFAULT", + ].join(" "); + + const separator = "-".repeat(header.length + 10); + + const allRows: { name: string; description: string; default: string }[] = []; + for (const opt of CONFIG_REFERENCE) { + allRows.push({ name: opt.name, description: opt.description, default: opt.default }); + + // Auto-expand array item properties (e.g., codebases[].*) + if (opt.items?.properties) { + expandSchema(`${opt.name}[]`, opt.items, allRows); + } + // Auto-expand additionalProperties (e.g., gitBranches..*) + if (opt.additionalProperties && typeof opt.additionalProperties === "object" && opt.additionalProperties.properties) { + expandSchema(`${opt.name}.`, opt.additionalProperties as Record, allRows); + } + // Auto-expand named properties (e.g., gitBranches.commonSpecificItems) + if (opt.properties) { + expandSchema(opt.name, opt, allRows); + } + } + + const rows = allRows.map((r) => + [r.name.padEnd(nameWidth), r.description.padEnd(descWidth), r.default].join(" ") + ); + + return [ + "wmill.yaml — Configuration Reference", + "", + "Full documentation: https://www.windmill.dev/docs/advanced/cli", + "", + separator, + header, + separator, + ...rows, + separator, + "", + 'Run "wmill init" to generate a wmill.yaml with commented examples.', + ].join("\n"); +} + +export function formatConfigReferenceJson(): string { + const clean = CONFIG_REFERENCE.map((opt) => ({ + name: opt.name, type: opt.type, default: opt.default, description: opt.description, + })); + return JSON.stringify(clean, null, 2); +} + +// ─── JSON Schema generator ────────────────────────────────────────────────── + +/** + * Generate a JSON Schema for wmill.yaml by stripping non-schema keys from CONFIG_REFERENCE. + */ +export function generateJsonSchema(): Record { + const properties: Record = {}; + for (const opt of CONFIG_REFERENCE) { + const entry: Record = {}; + for (const [k, v] of Object.entries(opt)) { + if (!NON_SCHEMA_KEYS.has(k) && k !== "name") { + entry[k] = v; + } + } + properties[opt.name] = entry; + } + return { + $schema: "http://json-schema.org/draft-07/schema#", + title: "wmill.yaml", + description: "Windmill CLI configuration file. Full reference: wmill config", + type: "object", + properties, + additionalProperties: false, + }; +} diff --git a/cli/src/commands/instance/instance.ts b/cli/src/commands/instance/instance.ts index 6b22d49b27..d95fe7269b 100644 --- a/cli/src/commands/instance/instance.ts +++ b/cli/src/commands/instance/instance.ts @@ -219,6 +219,22 @@ export async function pickInstance( prefix: opts.prefix ?? "custom", }; } + // Try to use the active workspace profile's remote as a fallback + if (instances.length < 1) { + try { + const ws = await getActiveWorkspace({}); + if (ws?.remote && ws?.token) { + const remote = ws.remote.endsWith("/") ? ws.remote.slice(0, -1) : ws.remote; + setClient(ws.token, remote); + return { + name: ws.name, + remote: ws.remote, + token: ws.token, + prefix: ws.name, + }; + } + } catch { /* ignore */ } + } if (!allowNew && instances.length < 1) { throw new Error("No instance found, please add one first"); } @@ -648,9 +664,27 @@ export async function getActiveInstance(opts: { } } -async function getConfig(opts: InstanceSyncOptions & { outputFile?: string }) { +async function getConfig(opts: InstanceSyncOptions & { outputFile?: string; showSecrets?: boolean }) { await pickInstance(opts, false); - const config = await wmill.getInstanceConfig(); + const config = await wmill.getInstanceConfig() as any; + + // In interactive mode, mask secrets by default and prompt + const hasSecrets = config?.global_settings?.license_key || config?.global_settings?.jwt_secret; + let showSecrets = opts.showSecrets ?? false; + if (!showSecrets && hasSecrets && process.stdout.isTTY && !opts.outputFile) { + log.warn("Config contains sensitive fields (license_key, jwt_secret). They are masked by default."); + log.warn("Use --show-secrets to include them, or press Y to show them now."); + showSecrets = await Confirm.prompt({ message: "Show secrets?", default: false }); + } else if (!process.stdout.isTTY || opts.outputFile) { + // Non-interactive or writing to file: always include secrets + showSecrets = true; + } + + if (!showSecrets && config?.global_settings) { + if (config.global_settings.license_key) config.global_settings.license_key = "***"; + if (config.global_settings.jwt_secret) config.global_settings.jwt_secret = "***"; + } + const yaml = yamlStringify(config as Record); if (opts.outputFile) { await writeFile(opts.outputFile, yaml, "utf-8"); @@ -786,6 +820,7 @@ const command = new Command() .command("get-config") .description("Dump the current instance config (global settings + worker configs) as YAML") .option("-o, --output-file ", "Write YAML to a file instead of stdout") + .option("--show-secrets", "Include sensitive fields (license key, JWT secret) without prompting") .option( "--instance ", "Name of the instance, override the active instance", diff --git a/cli/src/commands/job/job.ts b/cli/src/commands/job/job.ts new file mode 100644 index 0000000000..f542355fb7 --- /dev/null +++ b/cli/src/commands/job/job.ts @@ -0,0 +1,400 @@ +import { GlobalOptions } from "../../types.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { Command } from "@cliffy/command"; +import { Table } from "@cliffy/table"; +import { colors } from "@cliffy/ansi/colors"; +import * as log from "../../core/log.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import { formatTimestamp } from "../../utils/utils.ts"; + +function formatDuration(ms: number): string { + if (ms < 1000) return `${ms}ms`; + const seconds = Math.floor(ms / 1000); + if (seconds < 60) return `${seconds}s`; + const minutes = Math.floor(seconds / 60); + const remainingSeconds = seconds % 60; + if (minutes < 60) return `${minutes}m${remainingSeconds}s`; + const hours = Math.floor(minutes / 60); + const remainingMinutes = minutes % 60; + return `${hours}h${remainingMinutes}m`; +} + +function getJobStatus(job: any): string { + if (job.type === "QueuedJob") { + if (job.canceled) return colors.red("canceled"); + if (job.running) return colors.blue("running"); + return colors.yellow("queued"); + } + // CompletedJob + if (job.canceled) return colors.red("canceled"); + if (job.success) return colors.green("success"); + return colors.red("failure"); +} + +function getJobStatusPlain(job: any): string { + if (job.type === "QueuedJob") { + if (job.canceled) return "canceled"; + if (job.running) return "running"; + return "queued"; + } + if (job.canceled) return "canceled"; + if (job.success) return "success"; + return "failure"; +} + +async function list( + opts: GlobalOptions & { + json?: boolean; + scriptPath?: string; + createdBy?: string; + running?: boolean; + success?: boolean; + failed?: boolean; + limit?: number; + jobKinds?: string; + label?: string; + all?: boolean; + parent?: string; + isFlowStep?: boolean; + } +) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + // --failed is a convenience alias for --success false + let successFilter = opts.success; + if (opts.failed) successFilter = false; + + // When --all or --parent is used, include flow sub-job kinds too + const showSubJobs = opts.all || opts.parent; + const defaultJobKinds = showSubJobs + ? "script,flow,singlestepflow,flowscript,flowdependencies" + : "script,flow,singlestepflow"; + + const limit = Math.min(opts.limit ?? 30, 100); + const allJobs = await wmill.listJobs({ + workspace: workspace.workspaceId, + scriptPathExact: opts.scriptPath, + createdBy: opts.createdBy, + running: opts.running, + success: successFilter, + perPage: limit, + jobKinds: opts.jobKinds ?? defaultJobKinds, + label: opts.label, + hasNullParent: showSubJobs ? undefined : true, + parentJob: opts.parent, + isFlowStep: opts.isFlowStep, + }); + // API may return more than perPage — enforce limit client-side + const jobs = allJobs.slice(0, limit); + + if (opts.json) { + console.log(JSON.stringify(jobs)); + } else { + if (jobs.length === 0) { + log.info("No jobs found."); + return; + } + new Table() + .header(["ID", "Status", "Script/Flow", "Created By", "Duration", "Created At"]) + .padding(2) + .border(true) + .body( + jobs.map((j: any) => [ + j.id, + getJobStatus(j), + j.script_path ?? j.raw_code?.substring(0, 30) ?? "-", + j.created_by ?? j.email ?? "-", + j.duration_ms != null ? formatDuration(j.duration_ms) : (j.running ? "running" : "-"), + j.created_at ? formatTimestamp(j.created_at) : "-", + ]) + ) + .render(); + log.info(`\nShowing ${jobs.length} job(s). Use --limit to show more.`); + } +} + +function getModuleStatusIcon(type: string, success?: boolean): string { + switch (type) { + case "Success": return colors.green("✓"); + case "Failure": return colors.red("✗"); + case "InProgress": return colors.blue("▶"); + case "WaitingForPriorSteps": return colors.dim("○"); + case "WaitingForEvents": return colors.yellow("⏳"); + default: return colors.dim("·"); + } +} + +function formatFlowSteps( + flowStatus: any, + rawFlow: any, +) { + const modules = flowStatus?.modules ?? []; + const rawModules = rawFlow?.modules ?? []; + + // Build summary map from raw_flow + const summaryMap = new Map(); + for (const mod of rawModules) { + if (mod.id && mod.summary) { + summaryMap.set(mod.id, mod.summary); + } + } + + console.log(colors.bold("\nSteps:")); + for (const mod of modules) { + const icon = getModuleStatusIcon(mod.type); + const summary = summaryMap.get(mod.id) ?? ""; + const label = summary ? `${mod.id}: ${summary}` : mod.id; + const jobId = mod.job ? colors.dim(mod.job) : ""; + const flowJobsDuration = mod.flow_jobs_duration; + + // For-loop modules: show parent line + iteration sub-lines + const flowJobs = mod.flow_jobs as string[] | undefined; + if (flowJobs && flowJobs.length > 0) { + // Total duration for the for-loop + const totalMs = flowJobsDuration?.duration_ms + ? (flowJobsDuration.duration_ms as number[]).reduce((a: number, b: number) => a + b, 0) + : undefined; + const durationStr = totalMs != null ? colors.dim(formatDuration(totalMs)) : ""; + console.log(` ${icon} ${label} ${durationStr}`); + + const flowJobsSuccess = (mod.flow_jobs_success ?? []) as boolean[]; + const durationMs = (flowJobsDuration?.duration_ms ?? []) as number[]; + for (let iter = 0; iter < flowJobs.length; iter++) { + const iterSuccess = flowJobsSuccess[iter]; + const iterIcon = iterSuccess === true ? colors.green("✓") + : iterSuccess === false ? colors.red("✗") + : colors.dim("·"); + const iterDur = durationMs[iter] != null ? colors.dim(formatDuration(durationMs[iter])) : ""; + const iterJobId = colors.dim(flowJobs[iter]); + console.log(` ${iterIcon} iteration ${iter} ${iterJobId} ${iterDur}`); + } + } else { + // Regular step + const durationStr = mod.duration_ms != null + ? colors.dim(formatDuration(mod.duration_ms)) + : ""; + console.log(` ${icon} ${label} ${jobId} ${durationStr}`); + } + } + + // Show hint for diving into step logs + const hasJobs = modules.some((m: any) => m.job); + if (hasJobs) { + console.log(colors.dim("\nUse 'wmill job logs ' for step logs")); + } +} + +async function get( + opts: GlobalOptions & { json?: boolean }, + id: string +) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const job = await wmill.getJob({ + workspace: workspace.workspaceId, + id, + }); + + if (opts.json) { + console.log(JSON.stringify(job)); + } else { + const j = job as any; + console.log(colors.bold("ID:") + " " + j.id); + console.log(colors.bold("Status:") + " " + getJobStatusPlain(j)); + console.log(colors.bold("Kind:") + " " + j.job_kind); + console.log(colors.bold("Script Path:") + " " + (j.script_path ?? "-")); + console.log(colors.bold("Created By:") + " " + (j.created_by ?? "-")); + console.log(colors.bold("Created At:") + " " + (j.created_at ? formatTimestamp(j.created_at) : "-")); + if (j.started_at) { + console.log(colors.bold("Started At:") + " " + formatTimestamp(j.started_at)); + } + if (j.duration_ms != null) { + console.log(colors.bold("Duration:") + " " + formatDuration(j.duration_ms)); + } + if (j.schedule_path) { + console.log(colors.bold("Schedule:") + " " + j.schedule_path); + } + + // Flow: show hierarchical step status + const isFlow = j.job_kind === "flow" || j.job_kind === "flowpreview"; + if (isFlow && j.flow_status) { + formatFlowSteps(j.flow_status, j.raw_flow); + } + + if (j.result !== undefined) { + console.log(colors.bold("\nResult:")); + console.log(JSON.stringify(j.result, null, 2)); + } + } +} + +async function result( + opts: GlobalOptions, + id: string +) { + log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const jobResult = await wmill.getCompletedJobResult({ + workspace: workspace.workspaceId, + id, + }); + + console.log(JSON.stringify(jobResult)); +} + +async function logs( + opts: GlobalOptions, + id: string +) { + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + // Check if this is a flow job — if so, aggregate all step logs + try { + const job = await wmill.getJob({ + workspace: workspace.workspaceId, + id, + }); + const j = job as any; + const jobKind = j.job_kind; + if ((jobKind === "flow" || jobKind === "flowpreview") && j.flow_status?.modules) { + const modules = j.flow_status.modules; + const rawModules = j.raw_flow?.modules ?? []; + const summaryMap = new Map(); + for (const mod of rawModules) { + if (mod.id && mod.summary) summaryMap.set(mod.id, mod.summary); + } + + // Strip the "to remove ansi colors" hint that appears in each step's logs + const stripHint = (text: string) => + text.replace(/^to remove ansi colors.*\n?/gm, ""); + + let hasLogs = false; + for (const mod of modules) { + const summary = summaryMap.get(mod.id) ?? ""; + const label = summary ? `${mod.id}: ${summary}` : mod.id; + + // For-loop modules: get logs for each iteration + const flowJobs = mod.flow_jobs as string[] | undefined; + if (flowJobs && flowJobs.length > 0) { + for (let iter = 0; iter < flowJobs.length; iter++) { + try { + const stepLogs = await wmill.getJobLogs({ + workspace: workspace.workspaceId, + id: flowJobs[iter], + }); + if (stepLogs) { + console.log(colors.bold.cyan(`\n====== ${label} (iteration ${iter}) ======`)); + console.log(stripHint(stepLogs)); + hasLogs = true; + } + } catch { /* step may not exist yet */ } + } + } else if (mod.job) { + // Regular step + try { + const stepLogs = await wmill.getJobLogs({ + workspace: workspace.workspaceId, + id: mod.job, + }); + if (stepLogs) { + console.log(colors.bold.cyan(`\n====== ${label} ======`)); + console.log(stripHint(stepLogs)); + hasLogs = true; + } + } catch { /* step may not exist yet */ } + } + } + + if (!hasLogs) { + log.info("No logs available for this flow's steps."); + } + return; + } + } catch { + // If we can't get the job info, proceed with trying to get logs anyway + } + + const jobLogs = await wmill.getJobLogs({ + workspace: workspace.workspaceId, + id, + }); + + if (jobLogs == null || jobLogs === "") { + log.info("No logs available for this job."); + } else { + // Strip the hint if the API already includes it, then print it once to stderr + const stripped = jobLogs.replace(/^to remove ansi colors.*\n?/gm, ""); + console.error("to remove ansi colors, use: | sed 's/\\x1B\\[[0-9;]\\{1,\\}[A-Za-z]//g'"); + console.log(stripped); + } +} + +async function cancel( + opts: GlobalOptions & { reason?: string }, + id: string +) { + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + await wmill.cancelQueuedJob({ + workspace: workspace.workspaceId, + id, + requestBody: { + reason: opts.reason ?? "Canceled via CLI", + }, + }); + + log.info(colors.green(`Job ${id} canceled.`)); +} + +// Shared list options to avoid repetition between default action and list subcommand +const listOptions = (cmd: Command) => + cmd + .option("--json", "Output as JSON (for piping to jq)") + .option("--script-path ", "Filter by exact script/flow path") + .option("--created-by ", "Filter by creator username") + .option("--running", "Show only running jobs") + .option("--failed", "Show only failed jobs") + .option("--success ", "Filter by success status (true/false)") + .option("--limit ", "Number of jobs to return (default 30, max 100)") + .option("--job-kinds ", "Filter by job kinds (default: script,flow,singlestepflow)") + .option("--label ", "Filter by job label") + .option("--all", "Include sub-jobs (flow steps). By default only top-level jobs are shown") + .option("--parent ", "Filter by parent job ID (show sub-jobs of a specific flow)") + .option("--is-flow-step", "Show only flow step jobs"); + +const command = listOptions(new Command() + .description("Manage jobs (list, inspect, cancel)")) + .action(list as any) + .command("list", listOptions(new Command().description("List recent jobs"))) + .action(list as any) + .command("get", "Get job details. For flows: shows step tree with sub-job IDs") + .arguments("") + .option("--json", "Output as JSON (for piping to jq)") + .action(get as any) + .command("result", "Get the result of a completed job (machine-friendly)") + .arguments("") + .action(result as any) + .command("logs", "Get job logs. For flows: aggregates all step logs") + .arguments("") + .action(logs as any) + .command("cancel", "Cancel a running or queued job") + .arguments("") + .option("--reason ", "Reason for cancellation") + .action(cancel as any); + +export default command; diff --git a/cli/src/commands/lint/lint.ts b/cli/src/commands/lint/lint.ts index 62a491eee1..c726bff818 100644 --- a/cli/src/commands/lint/lint.ts +++ b/cli/src/commands/lint/lint.ts @@ -625,7 +625,13 @@ export async function runLint( throw new Error(`Path is not a directory: ${targetDirectory}`); } - const ignore = await ignoreF(mergedOpts); + // When the user specifies a subdirectory (that doesn't contain wmill.yaml), + // skip include/exclude filters since they're relative to the project root. + const isSubdirectory = explicitTargetDirectory && + !(await stat(path.join(targetDirectory, "wmill.yaml")).catch(() => null)); + const ignore = isSubdirectory + ? (_p: string, _isDir: boolean) => false + : await ignoreF(mergedOpts); const root = await FSFSElement(targetDirectory, [], false); const validator = new WindmillYamlValidator(); @@ -640,9 +646,10 @@ export async function runLint( if (entry.isDirectory || entry.ignored) { continue; } - scannedFiles += 1; const normalizedPath = normalizePath(entry.path); + + scannedFiles += 1; if (!YAML_FILE_REGEX.test(normalizedPath)) { continue; } @@ -742,7 +749,11 @@ export function printReport(report: LintReport, jsonOutput: boolean) { } } -async function lint(opts: LintOptions, directory?: string) { +async function lint(opts: LintOptions & { watch?: boolean }, directory?: string) { + if (opts.watch) { + await lintWatch(opts, directory); + return; + } try { const report = await runLint(opts, directory); printReport(report, !!opts.json); @@ -770,6 +781,37 @@ async function lint(opts: LintOptions, directory?: string) { } } +async function lintWatch(opts: LintOptions, directory?: string) { + const { watch } = await import("node:fs"); + const targetDir = directory ? path.resolve(process.cwd(), directory) : process.cwd(); + + log.info(colors.blue(`Watching ${targetDir} for changes... (Ctrl+C to stop)`)); + + async function runAndReport() { + try { + const report = await runLint(opts, directory); + // Clear screen for readability + process.stdout.write("\x1Bc"); + log.info(colors.gray(`[${new Date().toLocaleTimeString()}] Lint results:\n`)); + printReport(report, false); + } catch (error) { + log.error(error instanceof Error ? error.message : String(error)); + } + } + + await runAndReport(); + + let debounce: ReturnType | null = null; + watch(targetDir, { recursive: true }, (_event, filename) => { + if (!filename || !filename.toString().endsWith(".yaml") && !filename.toString().endsWith(".yml")) return; + if (debounce) clearTimeout(debounce); + debounce = setTimeout(runAndReport, 300); + }); + + // Keep the process alive + await new Promise(() => {}); +} + const command = new Command() .description( "Validate Windmill flow, schedule, and trigger YAML files in a directory", @@ -781,6 +823,7 @@ const command = new Command() "--locks-required", "Fail if scripts or flow inline scripts that need locks have no locks", ) + .option("-w, --watch", "Watch for file changes and re-lint automatically") .action(lint as any); export default command; diff --git a/cli/src/commands/resource-type/resource-type.ts b/cli/src/commands/resource-type/resource-type.ts index 0a5afe8dcb..96c8429eb4 100644 --- a/cli/src/commands/resource-type/resource-type.ts +++ b/cli/src/commands/resource-type/resource-type.ts @@ -88,6 +88,7 @@ async function push(opts: PushOptions, filePath: string, name: string) { } async function list(opts: GlobalOptions & { schema?: boolean; json?: boolean }) { + if (opts.json) log.setSilent(true); const workspace = await resolveWorkspace(opts); await requireLogin(opts); const res = await wmill.listResourceType({ @@ -96,6 +97,10 @@ async function list(opts: GlobalOptions & { schema?: boolean; json?: boolean }) if (opts.json) { console.log(JSON.stringify(res)); + } else if (res.length === 0) { + log.info("No custom resource types found in this workspace."); + log.info("Built-in types like 'postgresql', 'slack', 'mysql', etc. are available from the Windmill Hub."); + return; } else if (opts.schema) { new Table() .header(["Workspace", "Name", "Schema"]) diff --git a/cli/src/commands/resource/resource.ts b/cli/src/commands/resource/resource.ts index e28a479f13..e1c9052261 100644 --- a/cli/src/commands/resource/resource.ts +++ b/cli/src/commands/resource/resource.ts @@ -1,4 +1,4 @@ -import { stat, writeFile, readdir, readFile } from "node:fs/promises"; +import { mkdir, stat, writeFile, readdir, readFile } from "node:fs/promises"; import { stringify as yamlStringify } from "yaml"; import nodePath from "node:path"; @@ -155,6 +155,7 @@ async function push(opts: PushOptions, filePath: string, remotePath: string) { } async function list(opts: GlobalOptions & { json?: boolean }) { + if (opts.json) log.setSilent(true); const workspace = await resolveWorkspace(opts); await requireLogin(opts); let page = 0; @@ -202,6 +203,7 @@ async function newResource(opts: GlobalOptions, path: string) { resource_type: "", description: "", }; + await mkdir(nodePath.dirname(filePath), { recursive: true }); await writeFile(filePath, yamlStringify(template as Record), { flag: "wx", encoding: "utf-8", @@ -210,6 +212,7 @@ async function newResource(opts: GlobalOptions, path: string) { } async function get(opts: GlobalOptions & { json?: boolean }, path: string) { + if (opts.json) log.setSilent(true); const workspace = await resolveWorkspace(opts); await requireLogin(opts); const r = await wmill.getResource({ diff --git a/cli/src/commands/schedule/schedule.ts b/cli/src/commands/schedule/schedule.ts index c8582c5315..db57cabc64 100644 --- a/cli/src/commands/schedule/schedule.ts +++ b/cli/src/commands/schedule/schedule.ts @@ -1,4 +1,5 @@ -import { stat, writeFile } from "node:fs/promises"; +import { mkdir, stat, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; import { stringify as yamlStringify } from "yaml"; import { Command } from "@cliffy/command"; @@ -8,6 +9,7 @@ import * as log from "../../core/log.ts"; import { sep as SEP } from "node:path"; import { requireLogin } from "../../core/auth.ts"; import { resolveWorkspace, validatePath } from "../../core/context.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; import * as wmill from "../../../gen/services.gen.ts"; import { @@ -29,6 +31,7 @@ export interface ScheduleFile { } async function list(opts: GlobalOptions & { json?: boolean }) { + if (opts.json) log.setSilent(true); const workspace = await resolveWorkspace(opts); await requireLogin(opts); @@ -60,7 +63,7 @@ async function newSchedule(opts: GlobalOptions, path: string) { if (e.message?.startsWith("File already exists")) throw e; } const template: ScheduleFile = { - schedule: "0 */6 * * *", + schedule: "0 0 */6 * * *", on_failure: "", script_path: "", args: {}, @@ -68,6 +71,7 @@ async function newSchedule(opts: GlobalOptions, path: string) { is_flow: false, enabled: false, }; + await mkdir(dirname(filePath), { recursive: true }); await writeFile(filePath, yamlStringify(template as Record), { flag: "wx", encoding: "utf-8", @@ -76,6 +80,7 @@ async function newSchedule(opts: GlobalOptions, path: string) { } async function get(opts: GlobalOptions & { json?: boolean }, path: string) { + if (opts.json) log.setSilent(true); const workspace = await resolveWorkspace(opts); await requireLogin(opts); const s = await wmill.getSchedule({ @@ -162,6 +167,34 @@ export async function pushSchedule( } } +async function enable(opts: GlobalOptions, path: string) { + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + await wmill.setScheduleEnabled({ + workspace: workspace.workspaceId, + path, + requestBody: { enabled: true }, + }); + + log.info(colors.green(`Schedule ${path} enabled.`)); +} + +async function disable(opts: GlobalOptions, path: string) { + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + await wmill.setScheduleEnabled({ + workspace: workspace.workspaceId, + path, + requestBody: { enabled: false }, + }); + + log.info(colors.yellow(`Schedule ${path} disabled.`)); +} + async function push(opts: GlobalOptions, filePath: string, remotePath: string) { const workspace = await resolveWorkspace(opts); await requireLogin(opts); @@ -205,6 +238,12 @@ const command = new Command() "push a local schedule spec. This overrides any remote versions." ) .arguments(" ") - .action(push as any); + .action(push as any) + .command("enable", "Enable a schedule") + .arguments("") + .action(enable as any) + .command("disable", "Disable a schedule") + .arguments("") + .action(disable as any); export default command; diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 10015db776..9ab5064658 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -23,10 +23,13 @@ import { import { Workspace } from "../workspace/workspace.ts"; import { + checkifMetadataUptodate, generateScriptMetadataInternal, getRawWorkspaceDependencies, parseMetadataFile, + readLockfile, } from "../../utils/metadata.ts"; +import { generateHash, validateRequiredArgs } from "../../utils/utils.ts"; import { WorkspaceDependenciesLanguage, ScriptLanguage, @@ -101,7 +104,7 @@ export function isFlowInlineScriptPath(filePath: string): boolean { return isFlowInlineScriptPathInternal(filePath); } -type PushOptions = GlobalOptions; +type PushOptions = GlobalOptions & { message?: string }; async function push(opts: PushOptions, filePath: string) { opts = await mergeConfigWithConfigFile(opts); const workspace = await resolveWorkspace(opts); @@ -122,13 +125,35 @@ async function push(opts: PushOptions, filePath: string) { } await requireLogin(opts); + + // Warn about metadata state before pushing + try { + const content = await readFile(filePath, "utf-8"); + const remotePath = removeExtensionToPath(filePath).replaceAll(SEP, "/"); + const contentHash = await generateHash(content + remotePath); + const conf = await readLockfile(); + const hasLockEntry = conf.locks && (conf.locks[remotePath] !== undefined || conf.locks[`${remotePath}.ts`] !== undefined); + if (!hasLockEntry) { + log.warn(colors.yellow( + `No metadata generated yet for ${filePath}. Run 'wmill generate-metadata' to generate schema and lock.` + )); + } else if (!(await checkifMetadataUptodate(remotePath, contentHash, conf))) { + log.warn(colors.yellow( + `Metadata for ${filePath} appears stale (content changed since last 'wmill generate-metadata').\n` + + `The schema and lock may not match the current code. Consider running 'wmill generate-metadata' first.` + )); + } + } catch { + // Don't block push if check fails + } + const codebases = await listSyncCodebases(opts as SyncOptions); await handleFile( filePath, workspace, [], - undefined, + opts.message, opts, await getRawWorkspaceDependencies(true), codebases @@ -494,6 +519,7 @@ export async function handleFile( const body = { ...requestBodyCommon, parent_hash: remote.hash, + auto_parent: true, }; const execTime = await createScript( bundleContent, @@ -806,6 +832,8 @@ export function filePathExtensionFromContentType( return ".java"; } else if (language === "ruby") { return ".rb"; + } else if (language === "rlang") { + return ".r"; // for related places search: ADD_NEW_LANG } else { throw new Error("Invalid language: " + language); @@ -837,6 +865,7 @@ export const exts = [ ".playbook.yml", ".java", ".rb", + ".r", // for related places search: ADD_NEW_LANG ]; @@ -857,6 +886,7 @@ async function list( json?: boolean; } ) { + if (opts.json) log.setSilent(true); const workspace = await resolveWorkspace(opts); await requireLogin(opts); @@ -919,41 +949,93 @@ async function run( }, path: string ) { + if (opts.silent) { + log.setSilent(true); + } const workspace = await resolveWorkspace(opts); await requireLogin(opts); const input = opts.data ? await resolve(opts.data) : {}; - const id = await wmill.runScriptByPath({ - workspace: workspace.workspaceId, - path, - requestBody: input, - }); + + // Validate required args against schema when no data provided + if (!opts.data) { + try { + const script = await wmill.getScriptByPath({ + workspace: workspace.workspaceId, + path, + }); + validateRequiredArgs(script.schema as Record); + } catch (e: any) { + if (e.message?.startsWith("Missing required")) throw e; + log.warn(`Could not fetch schema to validate args: ${e.message}`); + } + } + + let id: string; + try { + id = await wmill.runScriptByPath({ + workspace: workspace.workspaceId, + path, + requestBody: input, + }); + } catch (e: any) { + if (e?.status === 404) { + // Script might exist but have a lock/deployment error — check before giving up + try { + const script = await wmill.getScriptByPath({ + workspace: workspace.workspaceId, + path, + }); + if (script.lock_error_logs) { + throw new Error( + `Script '${path}' has a deployment error and cannot be run:\n${script.lock_error_logs}` + ); + } + } catch (lookupErr: any) { + if (lookupErr?.message?.includes("deployment error")) throw lookupErr; + // Re-throw non-404 lookup errors (e.g. auth/network issues) + if (lookupErr?.status && lookupErr.status !== 404) throw lookupErr; + } + throw new Error( + `Script '${path}' not found. Run 'wmill script list' to see available scripts.` + ); + } + throw e; + } if (!opts.silent) { await track_job(workspace.workspaceId, id); } - while (true) { + const MAX_RETRIES = 600; // ~60 seconds at 100ms intervals + let retries = 0; + while (retries < MAX_RETRIES) { try { - const result = - ( - await wmill.getCompletedJob({ - workspace: workspace.workspaceId, - id, - }) - ).result ?? {}; + const completedJob = await wmill.getCompletedJob({ + workspace: workspace.workspaceId, + id, + }); + if (completedJob.success === false) { + process.exitCode = 1; + } + + const result = completedJob.result ?? {}; if (opts.silent) { - console.log(result); + console.log(JSON.stringify(result)); } else { log.info(JSON.stringify(result, null, 2)); } break; } catch { + retries++; await new Promise((resolve) => setTimeout(resolve, 100)); } } + if (retries >= MAX_RETRIES) { + throw new Error(`Timed out waiting for job ${id} to complete`); + } } export async function track_job(workspace: string, id: string) { @@ -1050,6 +1132,7 @@ async function show(opts: GlobalOptions, path: string) { } async function get(opts: GlobalOptions & { json?: boolean }, path: string) { + if (opts.json) log.setSilent(true); const workspace = await resolveWorkspace(opts); await requireLogin(opts); const s = await wmill.getScriptByPath({ @@ -1086,7 +1169,10 @@ async function bootstrap( const scriptInitialCode = scriptBootstrapCode[resolvedLanguage]; if (scriptInitialCode === undefined) { - throw new Error("Language unknown"); + const validLanguages = Object.keys(scriptBootstrapCode).sort().join(", "); + throw new Error( + `Unknown language '${language}'. Valid languages: ${validLanguages}` + ); } const config = await readConfigFile(); @@ -1261,6 +1347,9 @@ async function preview( } & SyncOptions, filePath: string ) { + if (opts.silent) { + log.setSilent(true); + } opts = await mergeConfigWithConfigFile(opts); const workspace = await resolveWorkspace(opts); await requireLogin(opts); @@ -1464,13 +1553,50 @@ async function preview( } } +async function history( + opts: GlobalOptions & { json?: boolean }, + scriptPath: string +) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const versions = await wmill.getScriptHistoryByPath({ + workspace: workspace.workspaceId, + path: scriptPath, + }); + + if (opts.json) { + console.log(JSON.stringify(versions)); + } else { + if (versions.length === 0) { + log.info("No version history found for " + scriptPath); + return; + } + new Table() + .header(["#", "Hash", "Created At", "Deployment Message"]) + .padding(2) + .border(true) + .body( + versions.map((v, i) => [ + String(versions.length - i), + v.script_hash, + v.created_at ? new Date(v.created_at).toLocaleString() : "-", + v.deployment_msg ?? "-", + ]) + ) + .render(); + } +} + const command = new Command() .description("script related commands") - .option("--show-archived", "Enable archived scripts in output") + .option("--show-archived", "Show archived scripts instead of active ones") .option("--json", "Output as JSON (for piping to jq)") .action(list as any) .command("list", "list all scripts") - .option("--show-archived", "Enable archived scripts in output") + .option("--show-archived", "Show archived scripts instead of active ones") .option("--json", "Output as JSON (for piping to jq)") .action(list as any) .command( @@ -1478,6 +1604,7 @@ const command = new Command() "push a local script spec. This overrides any remote versions. Use the script file (.ts, .js, .py, .sh)" ) .arguments("") + .option("--message ", "Deployment message") .action(push as any) .command("get", "get a script's details") .arguments("") @@ -1538,6 +1665,13 @@ const command = new Command() "-e --excludes ", "Comma separated patterns to specify which file to NOT take into account." ) - .action(generateMetadata as any); + .action(generateMetadata as any) + .command( + "history", + "show version history for a script" + ) + .arguments("") + .option("--json", "Output as JSON (for piping to jq)") + .action(history as any); export default command; diff --git a/cli/src/commands/sync/pull.ts b/cli/src/commands/sync/pull.ts index 826c1de090..0b4da237e1 100644 --- a/cli/src/commands/sync/pull.ts +++ b/cli/src/commands/sync/pull.ts @@ -3,9 +3,72 @@ import { colors } from "@cliffy/ansi/colors"; import { Command } from "@cliffy/command"; import * as log from "../../core/log.ts"; import JSZip from "jszip"; +import { extract } from "tar-stream"; +import { Readable } from "node:stream"; import { Workspace } from "../workspace/workspace.ts"; import { getHeaders } from "../../utils/utils.ts"; +/** + * Adapter that wraps tar entries in a JSZip-compatible interface + * so ZipFSElement in sync.ts can consume it without changes. + */ +class TarAsZip { + files: Record }> = {}; + + constructor(entries: Map) { + for (const [name, entry] of entries) { + const content = entry.content; + this.files[name] = { + dir: entry.isDir, + name, + async(_type: "text") { + return content; + }, + }; + } + } + + /** Return a filtered view containing only entries under the given prefix, with relative paths. */ + folder(prefix: string): TarAsZip | null { + const normalized = prefix.endsWith("/") ? prefix : prefix + "/"; + const sub = new TarAsZip(new Map()); + for (const [name, file] of Object.entries(this.files)) { + if (name.startsWith(normalized)) { + const relative = name.slice(normalized.length); + if (relative) { + sub.files[relative] = { ...file, name: relative }; + } + } + } + return Object.keys(sub.files).length > 0 ? sub : null; + } +} + +async function parseTarResponse(response: Response): Promise { + const buffer = Buffer.from(await response.arrayBuffer()); + const entries = new Map(); + const ex = extract(); + + return new Promise((resolve, reject) => { + ex.on("entry", (header, stream, next) => { + const chunks: Buffer[] = []; + stream.on("data", (chunk: Buffer) => chunks.push(chunk)); + stream.on("end", () => { + entries.set(header.name, { + content: Buffer.concat(chunks).toString("utf-8"), + isDir: header.type === "directory", + }); + next(); + }); + stream.on("error", reject); + stream.resume(); + }); + ex.on("finish", () => resolve(new TarAsZip(entries))); + ex.on("error", reject); + Readable.from(buffer).pipe(ex); + }); +} + export async function downloadZip( workspace: Workspace, plainSecrets: boolean | undefined, @@ -21,7 +84,7 @@ export async function downloadZip( includeKey?: boolean, skipWorkspaceDependencies?: boolean, defaultTs?: "bun" | "deno" -): Promise { +): Promise { const requestHeaders = new Headers(); requestHeaders.set("Authorization", "Bearer " + workspace.token); requestHeaders.set("Content-Type", "application/octet-stream"); @@ -34,38 +97,51 @@ export async function downloadZip( } const includeWorkspaceDependenciesValue = !(skipWorkspaceDependencies ?? false); - const url = workspace.remote + - "api/w/" + - workspace.workspaceId + - `/workspaces/tarball?archive_type=zip&plain_secret=${plainSecrets ?? false + const baseParams = `&plain_secret=${plainSecrets ?? false }&skip_variables=${skipVariables ?? false}&skip_resources=${skipResources ?? false }&skip_secrets=${skipSecrets ?? false}&include_schedules=${includeSchedules ?? false }&include_triggers=${includeTriggers ?? false}&include_users=${includeUsers ?? false }&include_groups=${includeGroups ?? false}&include_settings=${includeSettings ?? false }&include_key=${includeKey ?? false}&include_workspace_dependencies=${includeWorkspaceDependenciesValue}&default_ts=${defaultTs ?? "bun"}&skip_resource_types=${skipResourceTypes ?? false}&settings_version=v2`; - const zipResponse = await fetch(url, { - headers: requestHeaders, - method: "GET", - } - ); + const baseUrl = workspace.remote + "api/w/" + workspace.workspaceId + "/workspaces/tarball?"; - if (!zipResponse.ok) { - const body = await zipResponse.text(); - if (zipResponse.status === 404 || body.includes("no rows returned")) { - log.info(colors.red(`Workspace '${workspace.workspaceId}' not found on ${workspace.remote}. Please check your --workspace and try again.`)); - } else { - log.info(colors.red(`Failed to request tarball from API: ${zipResponse.status} ${zipResponse.statusText}`)); - if (body) { - log.info(colors.red(body)); - } - } - return process.exit(1); - } else { - log.debug(`Downloaded zip/tarball successfully`); + // Try zip first (standard format), fall back to tar if zip is not supported + const zipUrl = baseUrl + "archive_type=zip" + baseParams; + const zipResponse = await fetch(zipUrl, { headers: requestHeaders, method: "GET" }); + + if (zipResponse.ok) { + log.debug("Downloaded zip archive successfully"); + const blob = await zipResponse.blob(); + return await JSZip.loadAsync((await blob.arrayBuffer()) as any); } - const blob = await zipResponse.blob(); - return await JSZip.loadAsync((await blob.arrayBuffer()) as any); + + const body = await zipResponse.text(); + + // If zip format is not supported (backend compiled without zip feature), try tar + if (zipResponse.status === 400 && body.includes("Invalid Archive Type")) { + log.debug("Zip archive not supported by backend, falling back to tar"); + const tarUrl = baseUrl + "archive_type=tar" + baseParams; + const tarResponse = await fetch(tarUrl, { headers: requestHeaders, method: "GET" }); + + if (tarResponse.ok) { + log.debug("Downloaded tar archive successfully"); + return await parseTarResponse(tarResponse); + } + + const tarBody = await tarResponse.text(); + log.info(colors.red(`Failed to request tarball from API: ${tarResponse.status} ${tarResponse.statusText}`)); + if (tarBody) log.info(colors.red(tarBody)); + return process.exit(1); + } + + if (zipResponse.status === 404 || body.includes("no rows returned")) { + log.info(colors.red(`Workspace '${workspace.workspaceId}' not found on ${workspace.remote}. Please check your --workspace and try again.`)); + } else { + log.info(colors.red(`Failed to request tarball from API: ${zipResponse.status} ${zipResponse.statusText}`)); + if (body) log.info(colors.red(body)); + } + return process.exit(1); } function stub(_opts: GlobalOptions & { override: boolean }, _dir: string) { diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index d62c39cba9..eba891ce1d 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -76,7 +76,7 @@ import { newRawAppPathAssigner, PathAssigner, } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; -import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; +import { extractInlineScripts as extractInlineScriptsForFlows, extractCurrentMapping } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; import { generateFlowLockInternal } from "../flow/flow_metadata.ts"; import { isExecutionModeAnonymous } from "../app/app.ts"; import { @@ -93,6 +93,8 @@ import { isAppMetadataFile, isRawAppMetadataFile, isRawAppFolderMetadataFile, + isAppFolderMetadataFile, + isFlowFolderMetadataFile, getDeleteSuffix, transformJsonPathToDir, getFolderSuffix, @@ -636,9 +638,16 @@ function ZipFSElement( let inlineScripts; try { const assigner = newPathAssigner(defaultTs, { skipInlineScriptSuffix: getNonDottedPaths() }); - inlineScripts = extractInlineScriptsForFlows( + // Preserve original !inline filenames from the flow to avoid phantom renames + const inlineMapping = extractCurrentMapping( flow.value.modules as any, {}, + flow.value.failure_module, + flow.value.preprocessor_module, + ); + inlineScripts = extractInlineScriptsForFlows( + flow.value.modules as any, + inlineMapping, SEP, defaultTs, assigner, @@ -647,7 +656,7 @@ function ZipFSElement( if (flow.value.failure_module) { inlineScripts.push(...extractInlineScriptsForFlows( [flow.value.failure_module], - {}, + inlineMapping, SEP, defaultTs, assigner, @@ -657,7 +666,7 @@ function ZipFSElement( if (flow.value.preprocessor_module) { inlineScripts.push(...extractInlineScriptsForFlows( [flow.value.preprocessor_module], - {}, + inlineMapping, SEP, defaultTs, assigner, @@ -1297,6 +1306,7 @@ export async function elementsToMap( "nu", "java", "rb", + "r", // for related places search: ADD_NEW_LANG ].includes(path.split(".").pop() ?? "") ) { @@ -1516,6 +1526,10 @@ async function compareDynFSElement( continue; } if (k.startsWith("dependencies/")) { + if (!workspaceDependenciesPathToLanguageAndFilename(k)) { + log.warn(`Skipping unrecognized workspace dependencies file: ${k}`); + continue; + } log.info(`Adding workspace dependencies file: ${k}`); } changes.push({ name: "added", path: k, content: v }); @@ -1985,9 +1999,15 @@ export async function pull( opts: GlobalOptions & SyncOptions & { repository?: string; promotion?: string; branch?: string }, ) { + if ((opts as any).jsonOutput) log.setSilent(true); const originalCliOpts = { ...opts }; opts = await mergeConfigWithConfigFile(opts); + // --include-secrets overrides skipSecrets from wmill.yaml + if ((originalCliOpts as any).includeSecrets) { + opts.skipSecrets = false; + } + // Validate branch configuration early (skipped when --branch is used) try { await validateBranchConfiguration(opts, opts.branch); @@ -2476,12 +2496,18 @@ function removeSuffix(str: string, suffix: string) { export async function push( opts: GlobalOptions & SyncOptions & { repository?: string; branch?: string }, ) { + if ((opts as any).jsonOutput) log.setSilent(true); // Save original CLI options before merging with config file const originalCliOpts = { ...opts }; // Load configuration from wmill.yaml and merge with CLI options opts = await mergeConfigWithConfigFile(opts); + // --include-secrets overrides skipSecrets from wmill.yaml + if ((originalCliOpts as any).includeSecrets) { + opts.skipSecrets = false; + } + // Validate branch configuration early (skipped when --branch is used) try { await validateBranchConfiguration(opts, opts.branch); @@ -2615,6 +2641,7 @@ export async function push( const tracker: ChangeTracker = await buildTracker(changes); + const autoRegenerate = !!(opts as any).autoMetadata; const staleScripts: string[] = []; const staleFlows: string[] = []; const staleApps: string[] = []; @@ -2624,7 +2651,7 @@ export async function push( change, workspace, opts, - true, + !autoRegenerate, // dryRun=false when --auto is set true, rawWorkspaceDependencies, codebases, @@ -2637,11 +2664,19 @@ export async function push( if (staleScripts.length > 0) { log.info(""); - log.warn( - "Stale scripts metadata found, you may want to update them using 'wmill script generate-metadata' before pushing:", - ); + if (autoRegenerate) { + log.info("Auto-regenerated metadata for stale scripts:"); + } else { + log.warn( + "Stale scripts metadata found, you may want to update them using 'wmill script generate-metadata' before pushing:", + ); + } for (const stale of staleScripts) { - log.warn(stale); + if (autoRegenerate) { + log.info(` ${stale}`); + } else { + log.warn(stale); + } } log.info(""); @@ -2650,7 +2685,7 @@ export async function push( for (const change of tracker.flows) { const stale = await generateFlowLockInternal( change, - true, + !autoRegenerate, // dryRun=false when --auto is set workspace, opts, false, @@ -2662,11 +2697,19 @@ export async function push( } if (staleFlows.length > 0) { - log.warn( - "Stale flows locks found, you may want to update them using 'wmill flow generate-locks' before pushing:", - ); + if (autoRegenerate) { + log.info("Auto-regenerated locks for stale flows:"); + } else { + log.warn( + "Stale flows locks found, you may want to update them using 'wmill flow generate-locks' before pushing:", + ); + } for (const stale of staleFlows) { - log.warn(stale); + if (autoRegenerate) { + log.info(` ${stale}`); + } else { + log.warn(stale); + } } log.info(""); } @@ -2675,7 +2718,7 @@ export async function push( const stale = await generateAppLocksInternal( change, false, - true, + !autoRegenerate, workspace, opts, true, @@ -2690,7 +2733,7 @@ export async function push( const stale = await generateAppLocksInternal( change, true, - true, + !autoRegenerate, workspace, opts, true, @@ -2702,15 +2745,46 @@ export async function push( } if (staleApps.length > 0) { - log.warn( - "Stale apps locks found, you may want to update them using 'wmill app generate-locks' before pushing:", - ); + if (autoRegenerate) { + log.info("Auto-regenerated locks for stale apps:"); + } else { + log.warn( + "Stale apps locks found, you may want to update them using 'wmill app generate-locks' before pushing:", + ); + } for (const stale of staleApps) { - log.warn(stale); + if (autoRegenerate) { + log.info(` ${stale}`); + } else { + log.warn(stale); + } } log.info(""); } + // Warn about local files for skipped types. Walks the in-memory DynFSElement tree + // (not a fresh disk scan), but does re-traverse it. Acceptable cost for a one-time check. + { + const skippedWarnings: string[] = []; + let scheduleCount = 0; + let triggerCount = 0; + for await (const entry of readDirRecursiveWithIgnore(() => false, local)) { + if (entry.isDirectory) continue; + if (!opts.includeSchedules && entry.path.endsWith(".schedule.yaml")) scheduleCount++; + if (!opts.includeTriggers && entry.path.endsWith("_trigger.yaml")) triggerCount++; + } + if (scheduleCount > 0) { + skippedWarnings.push(`Skipping ${scheduleCount} schedule file(s). Use --include-schedules or set includeSchedules: true in wmill.yaml`); + } + if (triggerCount > 0) { + skippedWarnings.push(`Skipping ${triggerCount} trigger file(s). Use --include-triggers or set includeTriggers: true in wmill.yaml`); + } + for (const warning of skippedWarnings) { + log.warn(warning); + } + if (skippedWarnings.length > 0) log.info(""); + } + await fetchRemoteVersion(workspace); log.info( @@ -2727,9 +2801,32 @@ export async function push( } } for (const folderName of folderNames) { - try { - await stat(path.join("f", folderName, "folder.meta.yaml")); - } catch { + const basePath = path.join("f", folderName, "folder.meta.yaml"); + const branchPath = getBranchSpecificPath( + `f/${folderName}/folder.meta.yaml`, + specificItems, + opts.branch, + ); + let found = false; + // Check branch-specific variant first (e.g. folder.dev.meta.yaml) + if (branchPath) { + try { + await stat(branchPath); + found = true; + } catch { + // fall through to base path check + } + } + // Then check base path + if (!found) { + try { + await stat(basePath); + found = true; + } catch { + // not found + } + } + if (!found) { missingFolders.push(folderName); } } @@ -3160,16 +3257,88 @@ export async function push( }); break; case "flow": - await wmill.deleteFlowByPath({ - workspace: workspaceId, - path: removeSuffix(target, getDeleteSuffix("flow", "json")), - }); + if (isFlowFolderMetadataFile(target)) { + // Metadata file deleted — delete the entire flow + await wmill.deleteFlowByPath({ + workspace: workspaceId, + path: removeSuffix(target, getDeleteSuffix("flow", "json")), + }); + } else { + // Inline script file deleted within flow folder + const flowFolder = extractFolderPath(target, "flow"); + let flowFolderExists = false; + if (flowFolder) { + try { + await stat(flowFolder); + flowFolderExists = true; + } catch { + // folder doesn't exist + } + } + if (flowFolderExists) { + // Re-push the entire flow so the backend gets the updated definition + await pushObj( + workspaceId, + target, + undefined, + undefined, + opts.plainSecrets ?? false, + alreadySynced, + opts.message, + ); + } else { + // Flow folder doesn't exist locally — delete on server + const remotePath = extractResourceName(target, "flow"); + if (remotePath) { + await wmill.deleteFlowByPath({ + workspace: workspaceId, + path: remotePath, + }); + } + } + } break; case "app": - await wmill.deleteApp({ - workspace: workspaceId, - path: removeSuffix(target, getDeleteSuffix("app", "json")), - }); + if (isAppFolderMetadataFile(target)) { + // Metadata file deleted — delete the entire app + await wmill.deleteApp({ + workspace: workspaceId, + path: removeSuffix(target, getDeleteSuffix("app", "json")), + }); + } else { + // Inline script file deleted within app folder + const appFolder = extractFolderPath(target, "app"); + let appFolderExists = false; + if (appFolder) { + try { + await stat(appFolder); + appFolderExists = true; + } catch { + // folder doesn't exist + } + } + if (appFolderExists) { + // Re-push the entire app so the backend gets the updated definition + await pushObj( + workspaceId, + target, + undefined, + undefined, + opts.plainSecrets ?? false, + alreadySynced, + opts.message, + ); + } else { + // App folder doesn't exist locally — delete on server + const remotePath = extractResourceName(target, "app"); + if (remotePath) { + await wmill.deleteApp({ + workspace: workspaceId, + path: remotePath, + }); + } + } + } break; case "raw_app": if (isRawAppFolderMetadataFile(target)) { @@ -3448,6 +3617,7 @@ const command = new Command() .option("--json", "Use JSON instead of YAML") .option("--skip-variables", "Skip syncing variables (including secrets)") .option("--skip-secrets", "Skip syncing only secrets variables") + .option("--include-secrets", "Include secrets in sync (overrides skipSecrets in wmill.yaml)") .option("--skip-resources", "Skip syncing resources") .option("--skip-resource-types", "Skip syncing resource types") .option("--skip-scripts", "Skip syncing scripts") @@ -3503,6 +3673,7 @@ const command = new Command() .option("--json", "Use JSON instead of YAML") .option("--skip-variables", "Skip syncing variables (including secrets)") .option("--skip-secrets", "Skip syncing only secrets variables") + .option("--include-secrets", "Include secrets in sync (overrides skipSecrets in wmill.yaml)") .option("--skip-resources", "Skip syncing resources") .option("--skip-resource-types", "Skip syncing resource types") .option("--skip-scripts", "Skip syncing scripts") @@ -3552,6 +3723,7 @@ const command = new Command() "--locks-required", "Fail if scripts or flow inline scripts that need locks have no locks", ) + .option("--auto-metadata", "Automatically regenerate stale metadata (locks and schemas) before pushing") .action(push as any); export default command; diff --git a/cli/src/commands/token/token.ts b/cli/src/commands/token/token.ts new file mode 100644 index 0000000000..d0c8cda76d --- /dev/null +++ b/cli/src/commands/token/token.ts @@ -0,0 +1,87 @@ +import { GlobalOptions } from "../../types.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { Command } from "@cliffy/command"; +import { Table } from "@cliffy/table"; +import { colors } from "@cliffy/ansi/colors"; +import * as log from "../../core/log.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import { formatTimestamp } from "../../utils/utils.ts"; + +async function list(opts: GlobalOptions & { json?: boolean }) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + await requireLogin(opts); + + const tokens = await wmill.listTokens({ + excludeEphemeral: true, + }); + + if (opts.json) { + console.log(JSON.stringify(tokens)); + } else { + if (tokens.length === 0) { + log.info("No tokens found."); + return; + } + new Table() + .header(["Prefix", "Label", "Created At", "Last Used", "Expiration"]) + .padding(2) + .border(true) + .body( + tokens.map((t) => [ + t.token_prefix, + t.label ?? "-", + formatTimestamp(t.created_at), + formatTimestamp(t.last_used_at), + t.expiration ? formatTimestamp(t.expiration) : "never", + ]) + ) + .render(); + } +} + +async function create( + opts: GlobalOptions & { + label?: string; + expiration?: string; + } +) { + opts = await mergeConfigWithConfigFile(opts); + await requireLogin(opts); + + const token = await wmill.createToken({ + requestBody: { + label: opts.label, + expiration: opts.expiration, + }, + }); + + console.log(token); +} + +async function deleteToken(opts: GlobalOptions, tokenPrefix: string) { + opts = await mergeConfigWithConfigFile(opts); + await requireLogin(opts); + + await wmill.deleteToken({ tokenPrefix }); + + log.info(colors.green(`Token with prefix '${tokenPrefix}' deleted.`)); +} + +const command = new Command() + .description("Manage API tokens") + .option("--json", "Output as JSON (for piping to jq)") + .action(list as any) + .command("list", "List API tokens") + .option("--json", "Output as JSON (for piping to jq)") + .action(list as any) + .command("create", "Create a new API token") + .option("--label ", "Token label") + .option("--expiration ", "Token expiration (ISO 8601 timestamp)") + .action(create as any) + .command("delete", "Delete a token by its prefix") + .arguments("") + .action(deleteToken as any); + +export default command; diff --git a/cli/src/commands/trigger/trigger.ts b/cli/src/commands/trigger/trigger.ts index 11f68bea96..e8a310a102 100644 --- a/cli/src/commands/trigger/trigger.ts +++ b/cli/src/commands/trigger/trigger.ts @@ -1,4 +1,5 @@ -import { stat, writeFile } from "node:fs/promises"; +import { mkdir, stat, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; import { stringify as yamlStringify } from "yaml"; import * as wmill from "../../../gen/services.gen.ts"; @@ -308,11 +309,20 @@ const triggerTemplates: Record> = { http_method: "get", is_async: false, requires_auth: true, + request_type: "sync", + authentication_method: "none", + is_static_website: false, + workspaced_route: false, + wrap_body: false, + raw_string: false, }, websocket: { script_path: "", is_flow: false, url: "", + filters: [], + can_return_message: false, + can_return_error_result: false, enabled: false, }, kafka: { @@ -321,6 +331,7 @@ const triggerTemplates: Record> = { kafka_resource_path: "", group_id: "", topics: [], + filters: [], enabled: false, }, nats: { @@ -328,6 +339,7 @@ const triggerTemplates: Record> = { is_flow: false, nats_resource_path: "", subjects: [], + use_jetstream: false, enabled: false, }, postgres: { @@ -342,28 +354,31 @@ const triggerTemplates: Record> = { script_path: "", is_flow: false, mqtt_resource_path: "", - topics: [], - subscribe_qos: 0, + subscribe_topics: [], enabled: false, }, sqs: { script_path: "", is_flow: false, - sqs_resource_path: "", queue_url: "", + aws_resource_path: "", + aws_auth_resource_type: "credentials", enabled: false, }, gcp: { script_path: "", is_flow: false, gcp_resource_path: "", - subscription_id: "", topic_id: "", + subscription_id: "", + delivery_type: "pull", + subscription_mode: "create_update", enabled: false, }, email: { script_path: "", is_flow: false, + local_part: "", enabled: false, }, }; @@ -387,6 +402,7 @@ async function newTrigger(opts: GlobalOptions & { kind: string }, path: string) if (e.message?.startsWith("File already exists")) throw e; } const template = triggerTemplates[kind]; + await mkdir(dirname(filePath), { recursive: true }); await writeFile(filePath, yamlStringify(template), { flag: "wx", encoding: "utf-8", @@ -394,7 +410,29 @@ async function newTrigger(opts: GlobalOptions & { kind: string }, path: string) log.info(colors.green(`Created ${filePath}`)); } +const TRIGGER_SKIP_FIELDS = new Set(["workspace_id", "extra_perms", "edited_by", "edited_at"]); + +function printTriggerDetails(trigger: any, kind: string) { + console.log(colors.bold("Path:") + " " + trigger.path); + console.log(colors.bold("Kind:") + " " + kind); + console.log(colors.bold("Enabled:") + " " + (trigger.enabled ?? trigger.mode ?? "-")); + console.log(colors.bold("Script Path:") + " " + (trigger.script_path ?? "")); + console.log(colors.bold("Is Flow:") + " " + (trigger.is_flow ? "true" : "false")); + // Show all other non-internal fields + for (const [key, value] of Object.entries(trigger)) { + if (["path", "enabled", "mode", "script_path", "is_flow"].includes(key)) continue; + if (TRIGGER_SKIP_FIELDS.has(key)) continue; + if (value === undefined || value === null || value === "") continue; + const display = Array.isArray(value) ? (value.length > 0 ? JSON.stringify(value) : "[]") : + typeof value === "object" ? JSON.stringify(value) : String(value); + if (display === "[]" || display === "{}") continue; + const label = key.replace(/_/g, " ").replace(/\b\w/g, c => c.toUpperCase()); + console.log(colors.bold(label + ":") + " " + display); + } +} + async function get(opts: GlobalOptions & { json?: boolean; kind?: string }, path: string) { + if (opts.json) log.setSilent(true); const workspace = await resolveWorkspace(opts); await requireLogin(opts); @@ -406,11 +444,7 @@ async function get(opts: GlobalOptions & { json?: boolean; kind?: string }, path if (opts.json) { console.log(JSON.stringify(trigger)); } else { - console.log(colors.bold("Path:") + " " + (trigger as any).path); - console.log(colors.bold("Kind:") + " " + opts.kind); - console.log(colors.bold("Enabled:") + " " + ((trigger as any).enabled ?? "-")); - console.log(colors.bold("Script Path:") + " " + ((trigger as any).script_path ?? "")); - console.log(colors.bold("Is Flow:") + " " + ((trigger as any).is_flow ? "true" : "false")); + printTriggerDetails(trigger as any, opts.kind); } return; } @@ -435,11 +469,7 @@ async function get(opts: GlobalOptions & { json?: boolean; kind?: string }, path if (opts.json) { console.log(JSON.stringify(trigger)); } else { - console.log(colors.bold("Path:") + " " + trigger.path); - console.log(colors.bold("Kind:") + " " + kind); - console.log(colors.bold("Enabled:") + " " + (trigger.enabled ?? "-")); - console.log(colors.bold("Script Path:") + " " + (trigger.script_path ?? "")); - console.log(colors.bold("Is Flow:") + " " + (trigger.is_flow ? "true" : "false")); + printTriggerDetails(trigger, kind); } return; } @@ -461,6 +491,7 @@ async function listOrEmpty(fn: () => Promise): Promise { } async function list(opts: GlobalOptions & { json?: boolean }) { + if (opts.json) log.setSilent(true); const workspace = await resolveWorkspace(opts); await requireLogin(opts); diff --git a/cli/src/commands/user/user.ts b/cli/src/commands/user/user.ts index 207958ecce..b5ed0e3196 100644 --- a/cli/src/commands/user/user.ts +++ b/cli/src/commands/user/user.ts @@ -530,7 +530,7 @@ const command = new Command() .command("remove", "Delete a user") .arguments("") .action(remove as any) - .command("create-token") + .command("create-token", "Create a new API token for the authenticated user") .option( "--email ", "Specify credentials to use for authentication. This will not be stored. It will only be used to exchange for a token with the API server, which will not be stored either.", diff --git a/cli/src/commands/variable/variable.ts b/cli/src/commands/variable/variable.ts index 21b7a69eba..d902b8f79f 100644 --- a/cli/src/commands/variable/variable.ts +++ b/cli/src/commands/variable/variable.ts @@ -1,4 +1,5 @@ -import { stat, writeFile } from "node:fs/promises"; +import { mkdir, stat, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; import { stringify as yamlStringify } from "yaml"; import { requireLogin } from "../../core/auth.ts"; @@ -20,6 +21,7 @@ import * as wmill from "../../../gen/services.gen.ts"; import { ListableVariable } from "../../../gen/types.gen.ts"; async function list(opts: GlobalOptions & { json?: boolean }) { + if (opts.json) log.setSilent(true); const workspace = await resolveWorkspace(opts); await requireLogin(opts); @@ -62,6 +64,7 @@ async function newVariable(opts: GlobalOptions, path: string) { is_secret: false, description: "", }; + await mkdir(dirname(filePath), { recursive: true }); await writeFile(filePath, yamlStringify(template as Record), { flag: "wx", encoding: "utf-8", @@ -70,6 +73,7 @@ async function newVariable(opts: GlobalOptions, path: string) { } async function get(opts: GlobalOptions & { json?: boolean }, path: string) { + if (opts.json) log.setSilent(true); const workspace = await resolveWorkspace(opts); await requireLogin(opts); const v = await wmill.getVariable({ @@ -214,10 +218,10 @@ async function add( undefined, { value, - is_secret: !opts.public && !opts.plainSecrets, + is_secret: !opts.public, description: "", }, - opts.plainSecrets ?? false + true // value from CLI is always plaintext — tell API not to treat it as pre-encrypted ); log.info(colors.bold.underline.green(`Variable ${remotePath} pushed`)); } diff --git a/cli/src/commands/workspace/fork.ts b/cli/src/commands/workspace/fork.ts index 619f29fa2c..92bb38b799 100644 --- a/cli/src/commands/workspace/fork.ts +++ b/cli/src/commands/workspace/fork.ts @@ -129,10 +129,16 @@ async function createWorkspaceFork( const newBranchName = `${WM_FORK_PREFIX}/${clonedBranchName}/${workspaceId}` log.info(`Created forked workspace ${trueWorkspaceId}. To start contributing to your fork, create and push edits to the branch \`${newBranchName}\` by using the command: - + \t`+colors.white(`git checkout -b ${newBranchName}`) + ` - -When doing operations on the forked workspace, it will use the remote setup in gitBranches for the branch it was forked from.`); + +When doing operations on the forked workspace, it will use the remote setup in gitBranches for the branch it was forked from. + +To merge changes back to the parent workspace, you can: + - Use the Merge UI from the forked workspace home page + - Deploy individual items via the Deploy to staging/prod UI + - Use git: ` + colors.white(`git checkout ${clonedBranchName} && git merge ${newBranchName} && wmill sync push`) + ` + See: https://www.windmill.dev/docs/advanced/workspace_forks`); } async function deleteWorkspaceFork( @@ -141,54 +147,69 @@ async function deleteWorkspaceFork( }, name: string, ) { + let forkWorkspaceId: string; + let token: string; + let remote: string; + let hasLocalProfile = false; + + // Try local profile first (existing behavior) const orgWorkspaces = await allWorkspaces(opts.configDir); - const idxOf = orgWorkspaces.findIndex((x) => x.name === name) ; - if (idxOf === -1) { - log.info( - colors.red.bold(`! Workspace profile ${name} does not exist locally`) - ); - log.info("available workspace profiles:"); - await list(opts); - return; - } + const idxOf = orgWorkspaces.findIndex((x) => x.name === name); - const workspace = orgWorkspaces[idxOf]; - - if (!workspace.workspaceId.startsWith(WM_FORK_PREFIX)) { + if (idxOf !== -1) { + const workspace = orgWorkspaces[idxOf]; + if (!workspace.workspaceId.startsWith(WM_FORK_PREFIX)) { throw new Error( `You can only delete forked workspaces where the workspace id starts with \`${WM_FORK_PREFIX}.\` Failed while attempting to delete \`${workspace.workspaceId}\``, ); + } + forkWorkspaceId = workspace.workspaceId; + token = workspace.token; + remote = workspace.remote; + hasLocalProfile = true; + } else { + // Fallback: resolve parent workspace from branch config and construct fork ID + const parentWorkspace = await tryResolveBranchWorkspace(opts); + if (!parentWorkspace) { + throw new Error( + "Could not resolve parent workspace. Make sure you are in a git repo with gitBranches configured in wmill.yaml, or create a local workspace profile for the fork.", + ); + } + forkWorkspaceId = name.startsWith(`${WM_FORK_PREFIX}-`) ? name : `${WM_FORK_PREFIX}-${name}`; + token = parentWorkspace.token; + remote = parentWorkspace.remote; } if (!opts.yes) { - const { Select } = await import("@cliffy/prompt/select"); - const choice = await Select.prompt({ - message: `Are you sure you want to delete the forked workspace with id: \`${workspace.workspaceId}\`? This action will delete the workspace `, - options: [ - { name: "Yes", value: "confirm" }, - { name: "No", value: "cancel" }, - ], - }); + const { Select } = await import("@cliffy/prompt/select"); + const choice = await Select.prompt({ + message: `Are you sure you want to delete the forked workspace \`${forkWorkspaceId}\`?`, + options: [ + { name: "Yes", value: "confirm" }, + { name: "No", value: "cancel" }, + ], + }); - if (choice === "cancel") { - log.info("Operation cancelled"); - return; - } + if (choice === "cancel") { + log.info("Operation cancelled"); + return; + } } - const remote = workspace.remote setClient( - workspace.token, + token, remote.endsWith("/") ? remote.substring(0, remote.length - 1) : remote ); const result = await wmill.deleteWorkspace({ - workspace: workspace.workspaceId + workspace: forkWorkspaceId }); log.info( - colors.green(`✅ Forked workspace '${workspace.workspaceId}' deleted successfully!\n${result}`), + colors.green(`✅ Forked workspace '${forkWorkspaceId}' deleted successfully!\n${result}`), ); - await removeWorkspace(name, false, opts); + if (hasLocalProfile) { + await removeWorkspace(name, false, opts); + } } export { createWorkspaceFork, deleteWorkspaceFork }; diff --git a/cli/src/commands/workspace/workspace.ts b/cli/src/commands/workspace/workspace.ts index d82290e743..1b5d287069 100644 --- a/cli/src/commands/workspace/workspace.ts +++ b/cli/src/commands/workspace/workspace.ts @@ -253,8 +253,12 @@ export async function add( "On that instance and with those credentials, the workspaces that you can access are:" ); const workspaces = await wmill.listWorkspaces(); - for (const workspace of workspaces) { - log.info(`- ${workspace.id} (name: ${workspace.name})`); + if (workspaces.length === 0) { + log.info(" (none)"); + } else { + for (const workspace of workspaces) { + log.info(`- ${workspace.id} (name: ${workspace.name})`); + } } process.exit(1); } @@ -411,31 +415,94 @@ async function whoami(_opts: GlobalOptions) { const whoamiInfo = await wmill.globalWhoami(); log.info(JSON.stringify(whoamiInfo, null, 2)); const activeName = await getActiveWorkspaceName(_opts); - log.info("Active: " + colors.green.bold(activeName || "none")); + const { getCurrentGitBranch, getOriginalBranchForWorkspaceForks } = await import("../../utils/git.ts"); + const branch = getCurrentGitBranch(); + const originalBranch = branch ? getOriginalBranchForWorkspaceForks(branch) : null; + if (originalBranch) { + const { resolveWorkspace } = await import("../../core/context.ts"); + try { + const ws = await resolveWorkspace(_opts); + log.info("Active: " + colors.green.bold(ws.workspaceId) + ` (fork of ${activeName || "unknown"})`); + } catch { + log.info("Active: " + colors.green.bold(activeName || "none") + " (fork branch)"); + } + } else { + log.info("Active: " + colors.green.bold(activeName || "none")); + } } async function listRemote(_opts: GlobalOptions) { - const { resolveWorkspace } = await import("../../core/context.ts"); - const workspace = await resolveWorkspace(_opts); - await requireLogin(_opts); + let remote: string; + + if (_opts.baseUrl && _opts.token && !_opts.workspace) { + // Allow listing workspaces with just --base-url and --token (no --workspace needed) + const { setClient } = await import("../../core/client.ts"); + remote = new URL(_opts.baseUrl).toString(); + setClient(_opts.token, remote.replace(/\/$/, "")); + } else { + const { resolveWorkspace } = await import("../../core/context.ts"); + const workspace = await resolveWorkspace(_opts); + await requireLogin(_opts); + remote = workspace.remote; + } + const userWorkspaces = await wmill.listUserWorkspaces(); + const hasForks = userWorkspaces.workspaces.some((x) => x.parent_workspace_id); + const headers = hasForks + ? ["id", "name", "username", "fork of", "disabled"] + : ["id", "name", "username", "disabled"]; + new Table() - .header(["id", "name", "username", "disabled"]) + .header(headers) .padding(2) .border(true) .body( - userWorkspaces.workspaces.map((x) => [ + userWorkspaces.workspaces.map((x) => { + const row = [ + x.id, + x.name, + x.username, + ]; + if (hasForks) row.push(x.parent_workspace_id ?? "-"); + row.push(x.disabled ? colors.red("true") : "false"); + return row; + }) + ) + .render(); + + log.info(`Remote: ${colors.bold(remote)}`); + log.info(`Logged in as: ${colors.green.bold(userWorkspaces.email)}`); +} + +async function listForks(_opts: GlobalOptions) { + const { resolveWorkspace } = await import("../../core/context.ts"); + const workspace = await resolveWorkspace(_opts); + await requireLogin(_opts); + + const userWorkspaces = await wmill.listUserWorkspaces(); + const forks = userWorkspaces.workspaces.filter((w) => w.parent_workspace_id); + + if (forks.length === 0) { + log.info("No forked workspaces found."); + return; + } + + new Table() + .header(["id", "name", "fork of", "username"]) + .padding(2) + .border(true) + .body( + forks.map((x) => [ x.id, x.name, + x.parent_workspace_id ?? "", x.username, - x.disabled ? colors.red("true") : "false", ]) ) .render(); log.info(`Remote: ${colors.bold(workspace.remote)}`); - log.info(`Logged in as: ${colors.green.bold(userWorkspaces.email)}`); } export async function getActiveWorkspaceOrFallback(opts: GlobalOptions) { @@ -566,8 +633,11 @@ const command = new Command() .command("list-remote") .description("List workspaces on the remote server that you have access to") .action(listRemote as any) + .command("list-forks") + .description("List forked workspaces on the remote server") + .action(listForks as any) .command("bind") - .description("Bind the current Git branch to the active workspace") + .description("Bind the current Git branch to the active workspace. This adds the branch to gitBranches in wmill.yaml so sync operations use the correct workspace for each branch.") .option("--branch, --env ", "Specify branch/environment (defaults to current)") .action((opts) => bind(opts as any, true)) .command("unbind") diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index e72aa14414..826b207323 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -57,6 +57,7 @@ export interface SyncOptions { variables?: string[]; resources?: string[]; triggers?: string[]; + schedules?: string[]; folders?: string[]; settings?: boolean; }; @@ -70,6 +71,7 @@ export interface SyncOptions { variables?: string[]; resources?: string[]; triggers?: string[]; + schedules?: string[]; folders?: string[]; settings?: boolean; }; @@ -83,6 +85,7 @@ export interface SyncOptions { variables?: string[]; resources?: string[]; triggers?: string[]; + schedules?: string[]; folders?: string[]; settings?: boolean; }; @@ -96,6 +99,7 @@ export interface SyncOptions { variables?: string[]; resources?: string[]; triggers?: string[]; + schedules?: string[]; folders?: string[]; settings?: boolean; }; @@ -191,15 +195,18 @@ export function getWmillYamlPath(): string | null { return findWmillYaml(); } -export async function readConfigFile(): Promise { +export async function readConfigFile(opts?: { warnIfMissing?: boolean }): Promise { + const warnIfMissing = opts?.warnIfMissing ?? true; try { // First, try to find wmill.yaml recursively const wmillYamlPath = findWmillYaml(); if (!wmillYamlPath) { - log.warn( - "No wmill.yaml found. Use 'wmill init' to bootstrap it." - ); + if (warnIfMissing) { + log.warn( + "No wmill.yaml found. Use 'wmill init' to bootstrap it." + ); + } return {}; } diff --git a/cli/src/core/context.ts b/cli/src/core/context.ts index 2b7a2266e2..b65cdcab49 100644 --- a/cli/src/core/context.ts +++ b/cli/src/core/context.ts @@ -262,8 +262,8 @@ export async function tryResolveBranchWorkspace( } } - // Read wmill.yaml to check for branch workspace configuration - const config = await readConfigFile(); + // Read wmill.yaml to check for branch workspace configuration (silent — just probing) + const config = await readConfigFile({ warnIfMissing: false }); const branchConfig = config.gitBranches?.[currentBranch]; // Check if branch has workspace configuration @@ -366,7 +366,7 @@ export async function tryResolveBranchWorkspace( selectedProfile.name = `${selectedProfile.name}/${workspaceIdIfForked}`; selectedProfile.workspaceId = workspaceIdIfForked; log.info( - `Inferred workspace id \`${workspaceId}\` from branch name because this is a workspace fork branch (\`${rawBranch}\`). ` + `Using fork workspace \`${workspaceIdIfForked}\` (parent: \`${workspaceId}\`) from branch \`${rawBranch}\`` ); } @@ -458,15 +458,16 @@ export async function resolveWorkspace( const branch = branchOverride ?? getCurrentGitBranch(); // Try explicit workspace flag first (should override branch-based resolution). Unless it's a - // forked workspace, that we detect through the branch name (only when not using branchOverride) + // forked workspace, that we detect through the branch name (only when not using branchOverride + // and --workspace was not explicitly provided) const res = await tryResolveWorkspace(opts); if (!res.isError) { const workspace = (res as { isError: false; value: Workspace }).value; - if (branchOverride || !branch || !branch.startsWith(WM_FORK_PREFIX)) { + if (branchOverride || opts.workspace || !branch || !branch.startsWith(WM_FORK_PREFIX)) { return workspace; } else { log.info( - `Found an active workspace \`${workspace.name}\` but the branch name indicates this is a forked workspace. Ignoring active workspace and trying to resolve the correct workspace from the branch name \`${branch}\`` + `Found an active workspace \`${workspace.name}\` but the branch name indicates this is a forked workspace. Ignoring active workspace and trying to resolve the correct workspace from the branch name \`${branch}\`. Use --workspace to override.` ); } } else if (opts.workspace) { @@ -549,7 +550,7 @@ export async function resolveWorkspace( } // If everything failed, show error - log.info(colors.red.bold("No workspace given and no default set.")); + log.info(colors.red.bold("No workspace given and no default set. Run 'wmill workspace add' to configure one.")); return process.exit(-1); } diff --git a/cli/src/core/log.ts b/cli/src/core/log.ts index d7bed9a0d4..034e13e3e1 100644 --- a/cli/src/core/log.ts +++ b/cli/src/core/log.ts @@ -1,4 +1,5 @@ let logLevel: "DEBUG" | "INFO" | "WARN" | "ERROR" = "INFO"; +let silentMode = false; const levels = { DEBUG: 0, INFO: 1, WARN: 2, ERROR: 3 }; @@ -6,19 +7,25 @@ export function setup(level: "DEBUG" | "INFO" | "WARN" | "ERROR") { logLevel = level; } +export function setSilent(silent: boolean) { + silentMode = silent; +} + export function debug(msg: unknown) { if (levels[logLevel] <= levels.DEBUG) console.log(`\x1b[90m${String(msg)}\x1b[39m`); } export function info(msg: unknown) { + if (silentMode) return; console.log(`\x1b[34m${String(msg)}\x1b[39m`); } export function warn(msg: unknown) { + if (silentMode) return; console.log(`\x1b[33m${String(msg)}\x1b[39m`); } export function error(msg: unknown) { - console.log(`\x1b[31m${String(msg)}\x1b[39m`); + console.error(`\x1b[31m${String(msg)}\x1b[39m`); } diff --git a/cli/src/core/specific_items.ts b/cli/src/core/specific_items.ts index cfd806d9c6..dca403e905 100644 --- a/cli/src/core/specific_items.ts +++ b/cli/src/core/specific_items.ts @@ -8,6 +8,7 @@ export interface SpecificItemsConfig { variables?: string[]; resources?: string[]; triggers?: string[]; + schedules?: string[]; folders?: string[]; settings?: boolean; } @@ -17,6 +18,7 @@ function getBranchSpecificTypes() { return { variable: '.variable.yaml', resource: '.resource.yaml', + schedule: '.schedule.yaml', // Generate trigger patterns from the list ...Object.fromEntries( TRIGGER_TYPES.map(t => [`${t}_trigger`, `.${t}_trigger.yaml`]) @@ -31,6 +33,13 @@ function isTriggerFile(path: string): boolean { return TRIGGER_TYPES.some(type => path.endsWith(`.${type}_trigger.yaml`)); } +/** + * Check if a path is a schedule file + */ +function isScheduleFile(path: string): boolean { + return path.endsWith('.schedule.yaml'); +} + /** * Extract the file type suffix from a path */ @@ -53,7 +62,7 @@ function getFileTypeSuffix(path: string): string | null { * Build regex pattern for all supported yaml file types */ function buildYamlTypePattern(): string { - const basicTypes = ['variable', 'resource']; + const basicTypes = ['variable', 'resource', 'schedule']; const triggerTypes = TRIGGER_TYPES.map(t => `${t}_trigger`); return `((${basicTypes.join('|')})|(${triggerTypes.join('|')}))`; } @@ -100,6 +109,9 @@ export function getSpecificItemsForCurrentBranch(config: SyncOptions, branchOver if (commonItems?.triggers) { merged.triggers = [...commonItems.triggers]; } + if (commonItems?.schedules) { + merged.schedules = [...commonItems.schedules]; + } if (commonItems?.folders) { merged.folders = [...commonItems.folders]; } @@ -117,6 +129,9 @@ export function getSpecificItemsForCurrentBranch(config: SyncOptions, branchOver if (branchItems?.triggers) { merged.triggers = [...(merged.triggers || []), ...branchItems.triggers]; } + if (branchItems?.schedules) { + merged.schedules = [...(merged.schedules || []), ...branchItems.schedules]; + } if (branchItems?.folders) { merged.folders = [...(merged.folders || []), ...branchItems.folders]; } @@ -157,6 +172,10 @@ export function isItemTypeConfigured(path: string, specificItems: SpecificItemsC return specificItems.triggers !== undefined; } + if (isScheduleFile(path)) { + return specificItems.schedules !== undefined; + } + if (path.endsWith('/folder.meta.yaml')) { return specificItems.folders !== undefined; } @@ -194,6 +213,11 @@ export function isSpecificItem(path: string, specificItems: SpecificItemsConfig return specificItems.triggers ? matchesPatterns(path, specificItems.triggers) : false; } + // Check for schedule files + if (isScheduleFile(path)) { + return specificItems.schedules ? matchesPatterns(path, specificItems.schedules) : false; + } + // Check for folder meta files if (path.endsWith('/folder.meta.yaml')) { if (specificItems.folders) { diff --git a/cli/src/guidance/skills.ts b/cli/src/guidance/skills.ts index 81c87dd05a..2417ed9c12 100644 --- a/cli/src/guidance/skills.ts +++ b/cli/src/guidance/skills.ts @@ -24,6 +24,7 @@ export const SKILLS: SkillMetadata[] = [ { name: "write-script-postgresql", description: "MUST use when writing PostgreSQL queries.", languageKey: "postgresql" }, { name: "write-script-powershell", description: "MUST use when writing PowerShell scripts.", languageKey: "powershell" }, { name: "write-script-python3", description: "MUST use when writing Python scripts.", languageKey: "python3" }, + { name: "write-script-rlang", description: "MUST use when writing R scripts.", languageKey: "rlang" }, { name: "write-script-rust", description: "MUST use when writing Rust scripts.", languageKey: "rust" }, { name: "write-script-snowflake", description: "MUST use when writing Snowflake queries.", languageKey: "snowflake" }, { name: "write-flow", description: "MUST use when creating flows." }, @@ -4113,6 +4114,107 @@ async def parallel(items, fn, concurrency: Optional[int] = None) # offset: Message offset to commit (from event['offset']) def commit_kafka_offsets(trigger_path: str, topic: str, partition: int, offset: int) -> None +`, + "write-script-rlang": `--- +name: write-script-rlang +description: MUST use when writing R scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, tell the user they can run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Do NOT run these commands yourself. Instead, inform the user that they should run them. + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# R + +## Structure + +Define a \`main\` function using \`<-\` or \`=\` assignment. Parameters become the script inputs: + +\`\`\`r +library(dplyr) +library(jsonlite) + +main <- function(x, name = "default", flag = TRUE) { + df <- tibble(x = x, name = name) + result <- df %>% mutate(greeting = paste("Hello", name)) + return(toJSON(result, auto_unbox = TRUE)) +} +\`\`\` + +**Important:** +- The \`main\` function is required +- Use \`library()\` to load packages — they are resolved and installed automatically +- \`jsonlite\` is always available (used internally for argument parsing) +- Return values must be JSON-serializable + +## Parameters + +R types map to Windmill types: +- \`numeric\` → float/int +- \`character\` → string +- \`logical\` → bool (use \`TRUE\`/\`FALSE\`) +- \`list\` → object/dict +- \`NULL\` → null + +Default values are inferred from the function signature: + +\`\`\`r +main <- function( + name, # required string + count = 10, # optional int, default 10 + verbose = FALSE # optional bool, default FALSE +) { + # ... +} +\`\`\` + +## Resources and Variables + +Use the built-in Windmill helpers (no import needed): + +\`\`\`r +main <- function() { + # Get a variable + api_key <- get_variable("f/my_folder/api_key") + + # Get a resource (returns a list) + db <- get_resource("f/my_folder/postgres_config") + host <- db$host + port <- db$port + + return(list(host = host, port = port)) +} +\`\`\` + +## Output + +Return any JSON-serializable value from \`main\`. The return value becomes the step result: + +\`\`\`r +main <- function(x) { + # Return a scalar + return(x + 1) + + # Or a list (becomes JSON object) + return(list(result = x + 1, status = "ok")) +} +\`\`\` + +## Annotations + +Control execution behavior with comment annotations: + +\`\`\`r +#renv_verbose = true # Show verbose renv output during resolution +#renv_install_verbose = true # Show verbose output during package installation +#sandbox = true # Run in nsjail sandbox (requires nsjail) +\`\`\` `, "write-script-rust": `--- name: write-script-rust @@ -4359,7 +4461,7 @@ Reference a specific resource using \`$res:\` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_use":{"type":"boolean","description":"If true, this step's result is deleted after use to save memory"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_use":{"type":"boolean","description":"If true, this step's result is deleted after use to save memory"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, "raw-app": `--- name: raw-app description: MUST use when creating raw apps. @@ -4999,6 +5101,23 @@ app related commands - \`--dry-run\` - Perform a dry run without making changes - \`--default-ts \` - Default TypeScript runtime (bun or deno) +### audit + +View audit logs (requires admin) + +**Subcommands:** + +- \`audit list\` - List audit log entries +- \`audit get \` - Get a specific audit log entry + - \`--json\` - Output as JSON (for piping to jq) + +### config + +Show all available wmill.yaml configuration options + +**Options:** +- \`--json\` - Output as JSON for programmatic consumption + ### dependencies workspace dependencies related commands @@ -5011,14 +5130,14 @@ workspace dependencies related commands ### dev -Launch a dev server that will spawn a webserver with HMR +Launch a dev server that watches for local file changes and auto-pushes them to the remote workspace. Provides live reload for scripts and flows during development. **Options:** - \`--includes \` - Filter paths givena glob pattern or path ### docs -Search Windmill documentation. Requires Enterprise Edition. +Search Windmill documentation. **Arguments:** \`\` @@ -5041,6 +5160,7 @@ flow related commands - \`flow get \` - get a flow's details - \`--json\` - Output as JSON (for piping to jq) - \`flow push \` - push a local flow spec. This overrides any remote versions. + - \`--message \` - Deployment message - \`flow run \` - run a flow by path. - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. - \`-s --silent\` - Do not ouput anything other then the final output. Useful for scripting. @@ -5056,9 +5176,13 @@ flow related commands - \`flow new \` - create a new empty flow - \`--summary \` - flow summary - \`--description \` - flow description -- \`flow bootstrap \` - create a new empty flow (alias for new +- \`flow bootstrap \` - create a new empty flow (alias for new) - \`--summary \` - flow summary - \`--description \` - flow description +- \`flow history \` - Show version history for a flow + - \`--json\` - Output as JSON (for piping to jq) +- \`flow show-version \` - Show a specific version of a flow + - \`--json\` - Output as JSON (for piping to jq) ### folder @@ -5121,6 +5245,25 @@ Manage git-sync settings between local wmill.yaml and Windmill backend - \`--yes\` - Skip interactive prompts and use default behavior - \`--promotion \` - Use promotionOverrides from the specified branch instead of regular overrides +### group + +Manage workspace groups + +**Options:** +- \`--json\` - Output as JSON (for piping to jq) + +**Subcommands:** + +- \`group list\` - List all groups in the workspace + - \`--json\` - Output as JSON (for piping to jq) +- \`group get \` - Get group details and members + - \`--json\` - Output as JSON (for piping to jq) +- \`group create \` - Create a new group + - \`--summary \` - Group summary/description +- \`group delete \` - Delete a group +- \`group add-user \` - Add a user to a group +- \`group remove-user \` - Remove a user from a group + ### hub Hub related commands. EXPERIMENTAL. INTERNAL USE ONLY. @@ -5176,8 +5319,23 @@ sync local with a remote instance or the opposite (push or pull) - \`instance whoami\` - Display information about the currently logged-in user - \`instance get-config\` - Dump the current instance config (global settings + worker configs) as YAML - \`-o, --output-file \` - Write YAML to a file instead of stdout + - \`--show-secrets\` - Include sensitive fields (license key, JWT secret) without prompting - \`--instance \` - Name of the instance, override the active instance +### job + +Manage jobs (list, inspect, cancel) + +**Subcommands:** + +- \`job list\` - List recent jobs +- \`job get \` - Get job details. For flows: shows step tree with sub-job IDs + - \`--json\` - Output as JSON (for piping to jq) +- \`job result \` - Get the result of a completed job (machine-friendly) +- \`job logs \` - Get job logs. For flows: aggregates all step logs +- \`job cancel \` - Cancel a running or queued job + - \`--reason \` - Reason for cancellation + ### jobs Pull completed and queued jobs from workspace @@ -5204,6 +5362,7 @@ Validate Windmill flow, schedule, and trigger YAML files in a directory - \`--json\` - Output results in JSON format - \`--fail-on-warn\` - Exit with code 1 when warnings are emitted - \`--locks-required\` - Fail if scripts or flow inline scripts that need locks have no locks +- \`-w, --watch\` - Watch for file changes and re-lint automatically ### queues @@ -5264,24 +5423,27 @@ schedule related commands - \`--json\` - Output as JSON (for piping to jq) - \`schedule new \` - create a new schedule locally - \`schedule push \` - push a local schedule spec. This overrides any remote versions. +- \`schedule enable \` - Enable a schedule +- \`schedule disable \` - Disable a schedule ### script script related commands **Options:** -- \`--show-archived\` - Enable archived scripts in output +- \`--show-archived\` - Show archived scripts instead of active ones - \`--json\` - Output as JSON (for piping to jq) **Subcommands:** - \`script list\` - list all scripts - - \`--show-archived\` - Enable archived scripts in output + - \`--show-archived\` - Show archived scripts instead of active ones - \`--json\` - Output as JSON (for piping to jq) -- \`script push \` - push a local script spec. This overrides any remote versions. Use the script file (.ts, .js, .py, .sh +- \`script push \` - push a local script spec. This overrides any remote versions. Use the script file (.ts, .js, .py, .sh) + - \`--message \` - Deployment message - \`script get \` - get a script's details - \`--json\` - Output as JSON (for piping to jq) -- \`script show \` - show a script's content (alias for get +- \`script show \` - show a script's content (alias for get) - \`script run \` - run a script by path - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. @@ -5291,16 +5453,18 @@ script related commands - \`script new \` - create a new script - \`--summary \` - script summary - \`--description \` - script description -- \`script bootstrap \` - create a new script (alias for new +- \`script bootstrap \` - create a new script (alias for new) - \`--summary \` - script summary - \`--description \` - script description -- \`script generate-metadata [script:file]\` - re-generate the metadata file updating the lock and the script schema (for flows, use \`wmill flow generate-locks\` +- \`script generate-metadata [script:file]\` - re-generate the metadata file updating the lock and the script schema (for flows, use \`wmill flow generate-locks\`) - \`--yes\` - Skip confirmation prompt - \`--dry-run\` - Perform a dry run without making changes - \`--lock-only\` - re-generate only the lock - \`--schema-only\` - re-generate only script schema - \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) - \`-e --excludes \` - Comma separated patterns to specify which file to NOT take into account. +- \`script history \` - show version history for a script + - \`--json\` - Output as JSON (for piping to jq) ### sync @@ -5315,6 +5479,7 @@ sync local with a remote workspaces or the opposite (push or pull) - \`--json\` - Use JSON instead of YAML - \`--skip-variables\` - Skip syncing variables (including secrets) - \`--skip-secrets\` - Skip syncing only secrets variables + - \`--include-secrets\` - Include secrets in sync (overrides skipSecrets in wmill.yaml) - \`--skip-resources\` - Skip syncing resources - \`--skip-resource-types\` - Skip syncing resource types - \`--skip-scripts\` - Skip syncing scripts @@ -5344,6 +5509,7 @@ sync local with a remote workspaces or the opposite (push or pull) - \`--json\` - Use JSON instead of YAML - \`--skip-variables\` - Skip syncing variables (including secrets) - \`--skip-secrets\` - Skip syncing only secrets variables + - \`--include-secrets\` - Include secrets in sync (overrides skipSecrets in wmill.yaml) - \`--skip-resources\` - Skip syncing resources - \`--skip-resource-types\` - Skip syncing resource types - \`--skip-scripts\` - Skip syncing scripts @@ -5369,6 +5535,23 @@ sync local with a remote workspaces or the opposite (push or pull) - \`--branch, --env \` - Override the current git branch/environment (works even outside a git repository) - \`--lint\` - Run lint validation before pushing - \`--locks-required\` - Fail if scripts or flow inline scripts that need locks have no locks + - \`--auto-metadata\` - Automatically regenerate stale metadata (locks and schemas) before pushing + +### token + +Manage API tokens + +**Options:** +- \`--json\` - Output as JSON (for piping to jq) + +**Subcommands:** + +- \`token list\` - List API tokens + - \`--json\` - Output as JSON (for piping to jq) +- \`token create\` - Create a new API token + - \`--label \` - Token label + - \`--expiration \` - Token expiration (ISO 8601 timestamp) +- \`token delete \` - Delete a token by its prefix ### trigger @@ -5399,7 +5582,7 @@ user related commands - \`--company \` - Specify to set the company of the new user. - \`--name \` - Specify to set the name of the new user. - \`user remove \` - Delete a user -- \`user create-token\` +- \`user create-token\` - Create a new API token for the authenticated user - \`--email \` - Specify credentials to use for authentication. This will not be stored. It will only be used to exchange for a token with the API server, which will not be stored either. - \`--password \` - Specify credentials to use for authentication. This will not be stored. It will only be used to exchange for a token with the API server, which will not be stored either. @@ -5467,7 +5650,8 @@ workspace related commands - \`workspace whoami\` - Show the currently active user - \`workspace list\` - List local workspace profiles - \`workspace list-remote\` - List workspaces on the remote server that you have access to -- \`workspace bind\` - Bind the current Git branch to the active workspace +- \`workspace list-forks\` - List forked workspaces on the remote server +- \`workspace bind\` - Bind the current Git branch to the active workspace. This adds the branch to gitBranches in wmill.yaml so sync operations use the correct workspace for each branch. - \`--branch, --env \` - Specify branch/environment (defaults to current) - \`workspace unbind\` - Remove workspace binding from the current Git branch - \`--branch, --env \` - Specify branch/environment (defaults to current) @@ -5738,6 +5922,13 @@ properties: key: type: string value: {} + filter_logic: + type: string + enum: + - and + - or + description: Logic to apply when evaluating filters. 'and' requires all filters + to match, 'or' requires any filter to match. auto_offset_reset: type: string enum: @@ -6265,6 +6456,13 @@ properties: value: {} description: Array of key-value filters to match incoming messages (only matching messages trigger the script) + filter_logic: + type: string + enum: + - and + - or + description: Logic to apply when evaluating filters. 'and' requires all filters + to match, 'or' requires any filter to match. initial_messages: type: array items: diff --git a/cli/src/main.ts b/cli/src/main.ts index 30dd5d17fd..22022acfe6 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -39,8 +39,13 @@ import queues from "./commands/queues/queues.ts"; import dependencies from "./commands/dependencies/dependencies.ts"; import init from "./commands/init/init.ts"; import jobs from "./commands/jobs/jobs.ts"; +import job from "./commands/job/job.ts"; +import group from "./commands/group/group.ts"; +import audit from "./commands/audit/audit.ts"; +import token from "./commands/token/token.ts"; import generateMetadata from "./commands/generate-metadata/generate-metadata.ts"; import docs from "./commands/docs/docs.ts"; +import config from "./commands/config/config.ts"; import { fetchVersion } from "./core/context.ts"; export { @@ -62,13 +67,18 @@ export { instance, dev, docs, + config, hubPull, pull, push, workspaceAdd, + job, + group, + audit, + token, }; -export const VERSION = "1.665.0"; +export const VERSION = "1.672.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; @@ -130,8 +140,13 @@ const command = new Command() .command("queues", queues) .command("dependencies", dependencies) .command("jobs", jobs) + .command("job", job) + .command("group", group) + .command("audit", audit) + .command("token", token) .command("generate-metadata", generateMetadata) .command("docs", docs) + .command("config", config) .command("version --version", "Show version information") .action(async (opts: any) => { console.log("CLI version: " + VERSION); @@ -215,11 +230,24 @@ async function main() { await command.parse(args); } catch (e) { if (e && typeof e === "object" && "name" in e && e.name === "ApiError") { - console.log( - "Server failed. " + (e as any).statusText + ": " + (e as any).body + const body = (e as any).body; + let bodyStr = typeof body === "object" && body !== null ? JSON.stringify(body) : String(body ?? ""); + // Strip backend source file references like (flows.rs:1400) or @scripts.rs:123:45 + bodyStr = bodyStr.replace(/\s*[@(]\w+\.rs:\d+[:\d]*\)?/g, ""); + log.error( + "Server failed. " + (e as any).statusText + ": " + bodyStr ); + } else if (e instanceof Error) { + log.error(e.message); + } else if (e !== undefined && e !== null) { + log.error(String(e)); } - throw e; + const isDebug = + process.argv.includes("--verbose") || process.argv.includes("--debug"); + if (isDebug) { + throw e; + } + process.exitCode = 1; } } diff --git a/cli/src/types.ts b/cli/src/types.ts index 8ba36a0ea0..bfa0a47b9c 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -157,17 +157,26 @@ export async function pushObj( const typeEnding = getTypeStrFromPath(p); if (typeEnding === "app") { - const appName = extractResourceName(p, "app")!; + const appName = extractResourceName(p, "app"); + if (!appName) { + throw new Error(`Could not extract app name from path: ${p}`); + } await pushApp(workspace, appName, buildFolderPath(appName, "app"), message); } else if (typeEnding === "raw_app") { - const rawAppName = extractResourceName(p, "raw_app")!; + const rawAppName = extractResourceName(p, "raw_app"); + if (!rawAppName) { + throw new Error(`Could not extract raw app name from path: ${p}`); + } await pushRawApp(workspace, rawAppName, buildFolderPath(rawAppName, "raw_app"), message); } else if (typeEnding === "folder") { await pushFolder(workspace, p, befObj, newObj); } else if (typeEnding === "variable") { await pushVariable(workspace, p, befObj, newObj, plainSecrets); } else if (typeEnding === "flow") { - const flowName = extractResourceName(p, "flow")!; + const flowName = extractResourceName(p, "flow"); + if (!flowName) { + throw new Error(`Could not extract flow name from path: ${p}`); + } await pushFlow(workspace, flowName, buildFolderPath(flowName, "flow"), message); } else if (typeEnding === "resource") { if (!alreadySynced.includes(p)) { @@ -291,6 +300,7 @@ export function getTypeStrFromPath( parsed.ext == ".nu" || parsed.ext == ".java" || parsed.ext == ".rb" || + parsed.ext == ".r" || // for related places search: ADD_NEW_LANG (parsed.ext == ".yml" && parsed.name.split(".").pop() == "playbook") ) { @@ -349,12 +359,16 @@ export function removeType(str: string, type: string) { const normalizedStr = path.normalize(str).replaceAll(SEP, "/"); if ( - !normalizedStr.endsWith("." + type + ".yaml") && - !normalizedStr.endsWith("." + type + ".json") + normalizedStr.endsWith("." + type + ".yaml") || + normalizedStr.endsWith("." + type + ".json") ) { - throw new Error(str + " does not end with ." + type + ".(yaml|json)"); + return normalizedStr.slice(0, normalizedStr.length - type.length - 6); } - return normalizedStr.slice(0, normalizedStr.length - type.length - 6); + // Accept clean paths without the type suffix (e.g. "f/folder/name" instead of "f/folder/name.schedule.yaml") + if (normalizedStr.includes("." + type)) { + log.debug(`Path '${str}' contains '.${type}' but doesn't end with '.${type}.(yaml|json)' — treating as clean path`); + } + return normalizedStr; } /** diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index 0e010a0cc5..ccc20f7bcd 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -868,6 +868,9 @@ export async function inferSchema( } else if (language === "ruby") { const { parse_ruby } = await loadParser("windmill-parser-wasm-ruby"); inferedSchema = JSON.parse(parse_ruby(content)); + } else if (language === "rlang") { + const { parse_r } = await loadParser("windmill-parser-wasm-r"); + inferedSchema = JSON.parse(parse_r(content)); // for related places search: ADD_NEW_LANG } else { throw new Error("Invalid language: " + language); diff --git a/cli/src/utils/resource_folders.ts b/cli/src/utils/resource_folders.ts index 1531314f04..f720fdc680 100644 --- a/cli/src/utils/resource_folders.ts +++ b/cli/src/utils/resource_folders.ts @@ -48,7 +48,7 @@ let _nonDottedPathsLogged = false; */ export function setNonDottedPaths(value: boolean): void { if (value && !_nonDottedPathsLogged) { - log.info("Using non-dotted paths (__flow, __app, __raw_app)"); + log.debug("Using non-dotted paths (__flow, __app, __raw_app)"); _nonDottedPathsLogged = true; } _nonDottedPaths = value; @@ -453,6 +453,28 @@ export function isRawAppFolderMetadataFile(p: string): boolean { ); } +/** + * Check if a path ends with a specific app metadata file + * (inside the folder, e.g., ".app/app.yaml" or "__app/app.yaml") + */ +export function isAppFolderMetadataFile(p: string): boolean { + return ( + p.endsWith(getMetadataPathSuffix("app", "yaml")) || + p.endsWith(getMetadataPathSuffix("app", "json")) + ); +} + +/** + * Check if a path ends with a specific flow metadata file + * (inside the folder, e.g., ".flow/flow.yaml" or "__flow/flow.yaml") + */ +export function isFlowFolderMetadataFile(p: string): boolean { + return ( + p.endsWith(getMetadataPathSuffix("flow", "yaml")) || + p.endsWith(getMetadataPathSuffix("flow", "json")) + ); +} + // ============================================================================ // Script Module Path Functions // ============================================================================ diff --git a/cli/src/utils/script_common.ts b/cli/src/utils/script_common.ts index 7f126b6b85..f314fe0e5d 100644 --- a/cli/src/utils/script_common.ts +++ b/cli/src/utils/script_common.ts @@ -20,6 +20,7 @@ export type ScriptLanguage = | "nu" | "ansible" | "ruby" + | "rlang" | "java"; // for related places search: ADD_NEW_LANG @@ -105,6 +106,8 @@ export function inferContentTypeFromFilePath( return "java"; } else if (contentPath.endsWith(".rb")) { return "ruby"; + } else if (contentPath.endsWith(".r")) { + return "rlang"; // for related places search: ADD_NEW_LANG } else { throw new Error( diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index 766b372a0a..eb8babfff3 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -107,6 +107,7 @@ export function getHeaders(): Record | undefined { export async function digestDir(path: string, conf: string) { const hashes: string = []; const entries = await readdir(path, { withFileTypes: true }); + entries.sort((a, b) => a.name.localeCompare(b.name)); for (const e of entries) { const npath = path + "/" + e.name; if (e.isFile()) { @@ -287,3 +288,24 @@ export function toCamel(s: string) { export function capitalize(str: string): string { return str.charAt(0).toUpperCase() + str.slice(1); } + +export function formatTimestamp(ts: string): string { + return new Date(ts).toISOString().replace("T", " ").substring(0, 19); +} + +/** + * Validate that required arguments are present when no -d data was provided. + * Fetches the schema from the API and checks required fields. + * @param schema - The JSON schema object from the script/flow definition + * @throws Error if required arguments are missing + */ +export function validateRequiredArgs( + schema: Record | undefined | null, +): void { + const required = (schema as { required?: string[] })?.required ?? []; + if (required.length > 0) { + throw new Error( + `Missing required arguments: ${required.join(", ")}.\nUse -d '{"${required[0]}": ...}' to provide input data.` + ); + } +} diff --git a/cli/src/utils/yaml.ts b/cli/src/utils/yaml.ts index 9ad247c1fd..52ec682067 100644 --- a/cli/src/utils/yaml.ts +++ b/cli/src/utils/yaml.ts @@ -1,9 +1,35 @@ -import { parse as yamlParse, type ParseOptions } from "yaml"; +import { parse as yamlParse } from "yaml"; +import type { ParseOptions, DocumentOptions, SchemaOptions, ToJSOptions, ScalarTag } from "yaml"; import { readFile } from "node:fs/promises"; -export async function yamlParseFile(path: string, options: ParseOptions = {}) { +// Custom YAML tags that resolve `!inline value` and `!inline_fileset value` +// back to their string-prefix form ("!inline value"). +// Without these, the yaml parser strips the tag and returns just the scalar, +// breaking the string-prefix-based !inline detection used throughout the CLI. +const inlineTag: ScalarTag = { + tag: "!inline", + resolve(value: string) { + return "!inline " + value; + }, +}; + +const inlineFilesetTag: ScalarTag = { + tag: "!inline_fileset", + resolve(value: string) { + return "!inline_fileset " + value; + }, +}; + +const WINDMILL_CUSTOM_TAGS: ScalarTag[] = [inlineTag, inlineFilesetTag]; + +type YamlParseOptions = ParseOptions & DocumentOptions & SchemaOptions & ToJSOptions; + +export async function yamlParseFile(path: string, options: YamlParseOptions = {}) { try { - return yamlParse(await readFile(path, "utf-8"), options); + return yamlParse(await readFile(path, "utf-8"), { + ...options, + customTags: [...WINDMILL_CUSTOM_TAGS, ...((options.customTags as ScalarTag[] | undefined) ?? [])], + }); } catch (e) { throw new Error(`Error parsing yaml ${path}`, { cause: e }); } @@ -12,10 +38,13 @@ export async function yamlParseFile(path: string, options: ParseOptions = {}) { export function yamlParseContent( path: string, content: string, - options: ParseOptions = {}, + options: YamlParseOptions = {}, ) { try { - return yamlParse(content, options); + return yamlParse(content, { + ...options, + customTags: [...WINDMILL_CUSTOM_TAGS, ...((options.customTags as ScalarTag[] | undefined) ?? [])], + }); } catch (e) { throw new Error(`Error parsing yaml ${path}`, { cause: e }); } diff --git a/cli/test/app_inline_script_delete.test.ts b/cli/test/app_inline_script_delete.test.ts new file mode 100644 index 0000000000..a33b912971 --- /dev/null +++ b/cli/test/app_inline_script_delete.test.ts @@ -0,0 +1,158 @@ +import { expect, test } from "bun:test"; +import { withTestBackend } from "./test_backend.ts"; +import { addWorkspace } from "../workspace.ts"; +import * as path from "node:path"; +import { writeFile, readdir, stat, rm } from "node:fs/promises"; +import { getFolderSuffix, getMetadataFileName } from "../src/utils/resource_folders.ts"; + +// ============================================================================= +// APP INLINE SCRIPT DELETION TESTS +// Regression tests for: deleting inline script files within .app/ folders +// during sync push should re-push the app, not crash with TypeError. +// ============================================================================= + +async function fileExists(filePath: string): Promise { + try { + await stat(filePath); + return true; + } catch { + return false; + } +} + +test("App: delete inline script file and push does not crash", async () => { + await withTestBackend(async (backend, tempDir) => { + const testWorkspace = { + remote: backend.baseUrl, + workspaceId: backend.workspace, + name: "app_inline_delete_test", + token: backend.token, + }; + await addWorkspace(testWorkspace, { + force: true, + configDir: backend.testConfigDir, + }); + + await writeFile( + `${tempDir}/wmill.yaml`, + `defaultTs: bun\nincludes:\n - "**"\nexcludes: []`, + "utf-8" + ); + + // Create an app with an inline script via the API + const appPath = "f/test/inline_delete_app"; + const inlineContent = `export async function main() {\n return "hello";\n}`; + + // Create the folder first + await backend.apiRequest!( + `/api/w/${backend.workspace}/folders/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: "test" }), + } + ).then((r) => r.text()); + + await backend.createAppWithInlineScript!(appPath, inlineContent, "bun"); + + // ========================================================================= + // STEP 1: Pull — get the app folder with inline script files + // ========================================================================= + const pullResult1 = await backend.runCLICommand( + ["sync", "pull", "--yes"], + tempDir, + "app_inline_delete_test" + ); + expect(pullResult1.code).toEqual(0); + + // Find the app folder and its inline script files + const appSuffix = getFolderSuffix("app"); + const appDir = path.join(tempDir, appPath + appSuffix); + expect(await fileExists(appDir)).toBeTruthy(); + + // List files in the app folder to find inline script files + const appFiles = await readdir(appDir); + const inlineScriptFiles = appFiles.filter( + (f) => f.endsWith(".ts") || f.endsWith(".js") + ); + expect(inlineScriptFiles.length).toBeGreaterThan(0); + + const inlineScriptPath = path.join(appDir, inlineScriptFiles[0]); + expect(await fileExists(inlineScriptPath)).toBeTruthy(); + + const metadataFile = getMetadataFileName("app", "yaml"); + const appYamlPath = path.join(appDir, metadataFile); + + // ========================================================================= + // STEP 2: Remove the inline script from app.yaml and delete the .ts file + // ========================================================================= + // Replace the inline script with a static text component (no inline scripts) + const updatedAppYaml = `summary: Test app with inline script +value: + type: app + grid: + - id: text1 + data: + type: textcomponent + componentInput: + type: static + value: hello world + hiddenInlineScripts: [] + css: {} + norefreshbar: false +policy: + on_behalf_of: null + on_behalf_of_email: null + triggerables: {} + execution_mode: viewer +`; + await writeFile(appYamlPath, updatedAppYaml, "utf-8"); + + // Delete the inline script file + await rm(inlineScriptPath); + expect(await fileExists(inlineScriptPath)).toBeFalsy(); + + // Also delete any lock files for the inline script + for (const f of appFiles) { + if (f.endsWith(".lock")) { + await rm(path.join(appDir, f)); + } + } + + // ========================================================================= + // STEP 3: Push — should succeed, NOT crash with TypeError + // ========================================================================= + const pushResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir, + "app_inline_delete_test" + ); + + // The critical assertion: push should not crash + expect(pushResult.code).toEqual(0); + + // ========================================================================= + // STEP 4: Verify by pulling again — inline script should be gone + // ========================================================================= + await rm(appDir, { recursive: true }); + + const pullResult2 = await backend.runCLICommand( + ["sync", "pull", "--yes"], + tempDir, + "app_inline_delete_test" + ); + expect(pullResult2.code).toEqual(0); + + // App should still exist + expect(await fileExists(appDir)).toBeTruthy(); + + // But no inline script files should be present + const finalFiles = await readdir(appDir); + const finalInlineScripts = finalFiles.filter( + (f) => + (f.endsWith(".ts") || f.endsWith(".js")) && + f.includes("inline_script") + ); + expect(finalInlineScripts.length).toEqual(0); + }); +}); diff --git a/cli/test/audit_token_commands.test.ts b/cli/test/audit_token_commands.test.ts new file mode 100644 index 0000000000..66cc0b752f --- /dev/null +++ b/cli/test/audit_token_commands.test.ts @@ -0,0 +1,189 @@ +/** + * Integration tests for `wmill audit` and `wmill token` commands. + */ + +import { expect, test, describe } from "bun:test"; +import { withTestBackend } from "./test_backend.ts"; +import { setupWorkspaceProfile, createRemoteScript } from "./new_commands_helpers.ts"; + +// ============================================================================= +// audit commands +// ============================================================================= + +describe("audit command", () => { + test("audit list returns valid JSON", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + await createRemoteScript(backend, `f/test/audit_test_${Date.now()}`); + + const result = await backend.runCLICommand( + ["audit", "list", "--json"], + tempDir + ); + + expect(result.code).toEqual(0); + const parsed = JSON.parse(result.stdout); + expect(Array.isArray(parsed)).toBe(true); + }); + }); + + test("audit list with filters", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + await createRemoteScript(backend, `f/test/audit_filter_${Date.now()}`); + + const result = await backend.runCLICommand( + ["audit", "list", "--json", "--operation", "scripts", "--limit", "5"], + tempDir + ); + + expect(result.code).toEqual(0); + const parsed = JSON.parse(result.stdout); + expect(Array.isArray(parsed)).toBe(true); + if (parsed.length > 0) { + expect(parsed[0].operation).toMatch(/^scripts/); + } + }); + }); + + test("audit list shows table or empty message", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + await createRemoteScript(backend, `f/test/audit_table_${Date.now()}`); + + const result = await backend.runCLICommand(["audit", "list"], tempDir); + + expect(result.code).toEqual(0); + const output = result.stdout; + const hasTable = output.includes("ID") && output.includes("Operation"); + const hasEmpty = output.includes("No audit logs found"); + expect(hasTable || hasEmpty).toBe(true); + }); + }); + + test("audit get returns specific entry", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + await createRemoteScript(backend, `f/test/audit_get_${Date.now()}`); + + const listResult = await backend.runCLICommand( + ["audit", "list", "--json", "--limit", "1"], + tempDir + ); + expect(listResult.code).toEqual(0); + const logs = JSON.parse(listResult.stdout); + if (logs.length === 0) return; + + const auditId = String(logs[0].id); + const getResult = await backend.runCLICommand( + ["audit", "get", auditId, "--json"], + tempDir + ); + + expect(getResult.code).toEqual(0); + const parsed = JSON.parse(getResult.stdout); + expect(parsed.id).toBe(logs[0].id); + }); + }); + + test("audit --help shows all subcommands", async () => { + await withTestBackend(async (backend, tempDir) => { + const result = await backend.runCLICommand(["audit", "--help"], tempDir); + const output = result.stdout + result.stderr; + expect(output).toContain("list"); + expect(output).toContain("get"); + }); + }); +}); + +// ============================================================================= +// token commands +// ============================================================================= + +describe("token command", () => { + test("token list returns valid JSON", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const result = await backend.runCLICommand( + ["token", "list", "--json"], + tempDir + ); + + expect(result.code).toEqual(0); + const parsed = JSON.parse(result.stdout); + expect(Array.isArray(parsed)).toBe(true); + }); + }); + + test("token list shows table output", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const result = await backend.runCLICommand(["token", "list"], tempDir); + + expect(result.code).toEqual(0); + expect(result.stdout).toContain("Prefix"); + expect(result.stdout).toContain("Label"); + }); + }); + + test("token create + delete lifecycle", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + // Create + const createResult = await backend.runCLICommand( + ["token", "create", "--label", "cli-test-token"], + tempDir + ); + expect(createResult.code).toEqual(0); + const newToken = createResult.stdout.trim(); + expect(newToken.length).toBeGreaterThan(10); + + // List and find it + const listResult = await backend.runCLICommand( + ["token", "list", "--json"], + tempDir + ); + expect(listResult.code).toEqual(0); + const tokens = JSON.parse(listResult.stdout); + const found = tokens.find((t: any) => t.label === "cli-test-token"); + expect(found).toBeDefined(); + + // Delete + const deleteResult = await backend.runCLICommand( + ["token", "delete", found.token_prefix], + tempDir + ); + expect(deleteResult.code).toEqual(0); + expect(deleteResult.stdout).toContain("deleted"); + }); + }); + + test("default action (wmill token) lists tokens", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const result = await backend.runCLICommand(["token", "--json"], tempDir); + + expect(result.code).toEqual(0); + const parsed = JSON.parse(result.stdout); + expect(Array.isArray(parsed)).toBe(true); + }); + }); + + test("token --help shows all subcommands", async () => { + await withTestBackend(async (backend, tempDir) => { + const result = await backend.runCLICommand(["token", "--help"], tempDir); + const output = result.stdout + result.stderr; + expect(output).toContain("list"); + expect(output).toContain("create"); + expect(output).toContain("delete"); + }); + }); +}); diff --git a/cli/test/cargo_backend.ts b/cli/test/cargo_backend.ts index a25a788f7e..692fc1619b 100644 --- a/cli/test/cargo_backend.ts +++ b/cli/test/cargo_backend.ts @@ -14,11 +14,13 @@ import { resolve, dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { statSync } from "node:fs"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdtemp, rm, readFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { createServer } from "node:net"; import { Subprocess } from "bun"; +const IS_LINUX = process.platform === "linux"; + export interface CargoBackendConfig { /** PostgreSQL connection string (without database name) */ postgresUrl?: string; @@ -193,6 +195,10 @@ export class CargoBackend { this.process = null; } + // Kill any child processes (e.g. the windmill binary spawned by cargo) + // by matching our unique database name in their environment + await this.killProcessesByDbName(); + // Drop the test database await this.dropDatabase(); @@ -304,6 +310,15 @@ export class CargoBackend { } } + /** + * Kill any processes whose environment contains our unique database name. + * This catches child processes (e.g. the windmill binary spawned by cargo run) + * that survive after the direct child is killed. + */ + private async killProcessesByDbName(): Promise { + await killWindmillProcessesByEnvMatch(this.dbName); + } + /** * Start the backend process using cargo run */ @@ -762,6 +777,90 @@ export class CargoBackend { } } +/** + * Kill windmill processes whose /proc/pid/environ contains the given pattern. + * Used by both per-test cleanup (match specific DB name) and stale cleanup (match any test DB). + */ +async function killWindmillProcessesByEnvMatch(pattern: string): Promise { + if (!IS_LINUX) return; + try { + const pgrepProc = Bun.spawn(["pgrep", "-f", "target/(debug|release)/windmill"], { + stdout: "pipe", stderr: "pipe", + }); + const output = await new Response(pgrepProc.stdout).text(); + await new Response(pgrepProc.stderr).text(); + await pgrepProc.exited; + + for (const pidStr of output.trim().split("\n").filter(Boolean)) { + const pid = Number(pidStr); + if (isNaN(pid)) continue; + try { + const environ = await readFile(`/proc/${pid}/environ`, "utf-8"); + if (environ.includes(pattern)) { + console.log(`Killing orphaned test backend process: ${pid}`); + process.kill(pid, "SIGKILL"); + } + } catch { + // Process exited or we lack permissions + } + } + } catch { + // pgrep not available or no matches + } +} + +/** + * Clean up stale test databases and orphaned backend processes from previous + * test runs that crashed or were killed without proper cleanup. + * + * Should be called before starting a new test backend. + */ +export async function cleanupStaleTestResources(postgresUrl?: string): Promise { + const baseUrl = postgresUrl || process.env["DATABASE_URL"] || "postgres://postgres:changeme@localhost:5432"; + const url = new URL(baseUrl); + url.pathname = ""; + url.search = ""; + const cleanBaseUrl = url.toString().replace(/\/$/, ""); + + // 1. Find and drop stale windmill_test_* databases + try { + const listProc = Bun.spawn(["psql", `${cleanBaseUrl}/postgres`, "-t", "-c", + `SELECT datname FROM pg_database WHERE datname LIKE 'windmill_test_%';` + ], { stdout: "pipe", stderr: "pipe" }); + const output = await new Response(listProc.stdout).text(); + await new Response(listProc.stderr).text(); + await listProc.exited; + + const staleDBs = output.trim().split("\n").map(s => s.trim()).filter(Boolean); + for (const db of staleDBs) { + // Only touch databases matching the expected naming pattern + if (!/^windmill_test_[a-z0-9_]+$/.test(db)) continue; + console.log(`Cleaning up stale test database: ${db}`); + const termProc = Bun.spawn(["psql", `${cleanBaseUrl}/postgres`, "-c", + `SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${db}' AND pid <> pg_backend_pid();` + ], { stdout: "pipe", stderr: "pipe" }); + await new Response(termProc.stdout).text(); + await new Response(termProc.stderr).text(); + await termProc.exited; + + const dropProc = Bun.spawn(["psql", `${cleanBaseUrl}/postgres`, "-c", + `DROP DATABASE IF EXISTS "${db}";` + ], { stdout: "pipe", stderr: "pipe" }); + await new Response(dropProc.stdout).text(); + await new Response(dropProc.stderr).text(); + await dropProc.exited; + } + if (staleDBs.length > 0) { + console.log(`Cleaned up ${staleDBs.length} stale test database(s)`); + } + } catch (err) { + console.warn(`Warning: Failed to clean up stale databases: ${err}`); + } + + // 2. Find and kill orphaned windmill processes from test runs + await killWindmillProcessesByEnvMatch("windmill_test_"); +} + // Global backend instance let globalCargoBackend: CargoBackend | null = null; diff --git a/cli/test/conf_branch_override.test.ts b/cli/test/conf_branch_override_unit.test.ts similarity index 100% rename from cli/test/conf_branch_override.test.ts rename to cli/test/conf_branch_override_unit.test.ts diff --git a/cli/test/elements_to_map_branch_specific.test.ts b/cli/test/elements_to_map_branch_specific_unit.test.ts similarity index 100% rename from cli/test/elements_to_map_branch_specific.test.ts rename to cli/test/elements_to_map_branch_specific_unit.test.ts diff --git a/cli/test/flow_notes_ordering_unit.test.ts b/cli/test/flow_notes_ordering_unit.test.ts new file mode 100644 index 0000000000..b5906cb548 --- /dev/null +++ b/cli/test/flow_notes_ordering_unit.test.ts @@ -0,0 +1,219 @@ +/** + * Unit tests for flow notes/groups preservation and YAML field ordering. + * + * Verifies that: + * - Notes and groups survive a round-trip through generate-metadata (#8641) + * - YAML output uses consistent field ordering via yamlOptions + * + * No backend required — tests the YAML parse/stringify layer. + */ + +import { expect, test, describe } from "bun:test"; +import { yamlParseContent } from "../src/utils/yaml.ts"; +import { stringify as yamlStringify } from "yaml"; +import { yamlOptions } from "../src/commands/sync/sync.ts"; + +const FLOW_WITH_NOTES = ` +summary: Sync item +description: '' +value: + modules: + - id: fetch + summary: Fetch product + value: + type: script + input_transforms: + connection: + type: static + value: some_resource + is_trigger: false + path: f/api/product_get + - id: map + summary: Map item + value: + type: script + input_transforms: + bc_item: + type: javascript + expr: flow_input.bc_item + is_trigger: false + path: f/mapping/item_to_product + notes: + - id: note-abc123 + type: group + color: blue + contained_node_ids: + - fetch + - map + locked: false + text: These steps must run together +schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object +`; + +const FLOW_WITH_GROUPS = ` +summary: Test flow +description: '' +value: + modules: + - id: a + value: + type: identity + groups: + - summary: My group + start_id: a + end_id: a + color: green +schema: + type: object +`; + +describe("flow notes preservation (#8641)", () => { + test("notes survive YAML round-trip with yamlOptions", () => { + const parsed = yamlParseContent("flow.yaml", FLOW_WITH_NOTES); + + // Verify notes were parsed + expect(parsed.value.notes).toBeDefined(); + expect(parsed.value.notes).toHaveLength(1); + expect(parsed.value.notes[0].id).toBe("note-abc123"); + expect(parsed.value.notes[0].color).toBe("blue"); + expect(parsed.value.notes[0].text).toBe("These steps must run together"); + + // Simulate the generate-metadata round-trip: + // 1. Backend returns a new value WITHOUT notes (like FlowValue does) + const backendResponse = { ...parsed.value }; + delete backendResponse.notes; + + // 2. CLI preserves notes (our fix) + const savedNotes = parsed.value.notes; + parsed.value = backendResponse; + if (savedNotes !== undefined) parsed.value.notes = savedNotes; + + // 3. Serialize back to YAML + const output = yamlStringify(parsed, yamlOptions); + + // 4. Re-parse and verify notes are intact + const reparsed = yamlParseContent("flow.yaml", output); + expect(reparsed.value.notes).toBeDefined(); + expect(reparsed.value.notes).toHaveLength(1); + expect(reparsed.value.notes[0].id).toBe("note-abc123"); + expect(reparsed.value.notes[0].color).toBe("blue"); + expect(reparsed.value.notes[0].contained_node_ids).toEqual(["fetch", "map"]); + expect(reparsed.value.notes[0].text).toBe("These steps must run together"); + }); + + test("groups survive YAML round-trip with yamlOptions", () => { + const parsed = yamlParseContent("flow.yaml", FLOW_WITH_GROUPS); + + expect(parsed.value.groups).toBeDefined(); + expect(parsed.value.groups).toHaveLength(1); + expect(parsed.value.groups[0].summary).toBe("My group"); + + // Simulate backend stripping groups + const backendResponse = { ...parsed.value }; + delete backendResponse.groups; + + const savedGroups = parsed.value.groups; + parsed.value = backendResponse; + if (savedGroups !== undefined) parsed.value.groups = savedGroups; + + const output = yamlStringify(parsed, yamlOptions); + const reparsed = yamlParseContent("flow.yaml", output); + expect(reparsed.value.groups).toBeDefined(); + expect(reparsed.value.groups).toHaveLength(1); + expect(reparsed.value.groups[0].summary).toBe("My group"); + }); + + test("flow without notes or groups is unaffected", () => { + const yaml = ` +summary: Simple flow +value: + modules: + - id: a + value: + type: identity +schema: + type: object +`; + const parsed = yamlParseContent("flow.yaml", yaml); + expect(parsed.value.notes).toBeUndefined(); + expect(parsed.value.groups).toBeUndefined(); + + // Simulate the save/restore logic with undefined + const savedNotes = parsed.value.notes; + const savedGroups = parsed.value.groups; + // Replace value (simulating backend response) + parsed.value = { ...parsed.value }; + if (savedNotes !== undefined) parsed.value.notes = savedNotes; + if (savedGroups !== undefined) parsed.value.groups = savedGroups; + + const output = yamlStringify(parsed, yamlOptions); + const reparsed = yamlParseContent("flow.yaml", output); + expect(reparsed.value.notes).toBeUndefined(); + expect(reparsed.value.groups).toBeUndefined(); + }); +}); + +describe("flow YAML field ordering", () => { + test("yamlOptions produces consistent field order for flow modules", () => { + // Simulate a flow value with fields in random order (like backend response) + const unordered = { + summary: "Test", + value: { + modules: [ + { + value: { type: "script", path: "f/test", is_trigger: false, input_transforms: {} }, + id: "step1", + summary: "Step 1", + }, + ], + }, + schema: { type: "object" }, + description: "", + }; + + const output = yamlStringify(unordered, yamlOptions); + + // With yamlOptions, 'id' should come before 'summary' and 'value' + // because prioritizeName gives "id" → "aa", "summary" → "ad", "value" → "ah" + // Note: YAML sequence items start with "- id:" on the first key + const lines = output.split("\n"); + const idLine = lines.findIndex((l) => /^\s*-?\s*id:/.test(l)); + const summaryLine = lines.findIndex((l, i) => i > idLine && /^\s+summary:/.test(l)); + + expect(idLine).toBeGreaterThan(-1); + expect(summaryLine).toBeGreaterThan(idLine); + }); + + test("yamlOptions produces same output regardless of input key order", () => { + const order1 = { + summary: "Flow", + description: "", + value: { modules: [{ id: "a", summary: "S", value: { type: "identity" } }] }, + schema: { type: "object" }, + }; + const order2 = { + schema: { type: "object" }, + value: { modules: [{ value: { type: "identity" }, summary: "S", id: "a" }] }, + description: "", + summary: "Flow", + }; + + const output1 = yamlStringify(order1, yamlOptions); + const output2 = yamlStringify(order2, yamlOptions); + expect(output1).toBe(output2); + }); + + test("notes field is preserved in correct position after modules", () => { + const parsed = yamlParseContent("flow.yaml", FLOW_WITH_NOTES); + const output = yamlStringify(parsed, yamlOptions); + + // 'modules' should appear before 'notes' in the output + const modulesIdx = output.indexOf("modules:"); + const notesIdx = output.indexOf("notes:"); + expect(modulesIdx).toBeGreaterThan(-1); + expect(notesIdx).toBeGreaterThan(-1); + expect(modulesIdx).toBeLessThan(notesIdx); + }); +}); diff --git a/cli/test/folder_missing_meta.test.ts b/cli/test/folder_missing_meta.test.ts index 0c95a9ef4f..429ef83b94 100644 --- a/cli/test/folder_missing_meta.test.ts +++ b/cli/test/folder_missing_meta.test.ts @@ -397,4 +397,39 @@ describe("sync push missing folder detection", () => { expect(output).not.toContain("Missing folder.meta.yaml"); }); }); + + test("no warning when branch-specific folder.meta.yaml exists", async () => { + await withIsolatedWorkspace(async ({ tempDir, runCLICommand }) => { + const uniqueId = Date.now(); + const folderName = `branchmeta${uniqueId}`; + + // wmill.yaml with branch-specific folders configured + await writeFile( + join(tempDir, "wmill.yaml"), + `defaultTs: bun\nincludes:\n - "**"\nexcludes: []\ngitBranches:\n dev:\n specificItems:\n folders:\n - "f/${folderName}"\n`, + "utf-8" + ); + + // Create folder with branch-specific meta only (no base folder.meta.yaml) + await mkdir(join(tempDir, "f", folderName), { recursive: true }); + await writeFile( + join(tempDir, "f", folderName, "folder.dev.meta.yaml"), + `summary: ""\ndisplay_name: "${folderName}"\nowners: []\nextra_perms: {}\n`, + "utf-8" + ); + await writeFile( + join(tempDir, "f", folderName, "test_script.ts"), + 'export async function main() { return "hello"; }', + "utf-8" + ); + + const result = await runCLICommand( + ["sync", "push", "--yes", "--branch", "dev", "--includes", `f/${folderName}/**`], + ); + + expect(result.code).toEqual(0); + const output = result.stdout + result.stderr; + expect(output).not.toContain("Missing folder.meta.yaml"); + }); + }); }); diff --git a/cli/test/generate_metadata.test.ts b/cli/test/generate_metadata_unit.test.ts similarity index 100% rename from cli/test/generate_metadata.test.ts rename to cli/test/generate_metadata_unit.test.ts diff --git a/cli/test/group_commands.test.ts b/cli/test/group_commands.test.ts new file mode 100644 index 0000000000..54f769c0a0 --- /dev/null +++ b/cli/test/group_commands.test.ts @@ -0,0 +1,138 @@ +/** + * Integration tests for `wmill group` commands: + * list, get, create, delete, add-user, remove-user + */ + +import { expect, test, describe } from "bun:test"; +import { withTestBackend } from "./test_backend.ts"; +import { setupWorkspaceProfile } from "./new_commands_helpers.ts"; + +describe("group command", () => { + test("group list returns valid JSON", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const result = await backend.runCLICommand( + ["group", "list", "--json"], + tempDir + ); + + expect(result.code).toEqual(0); + const parsed = JSON.parse(result.stdout); + expect(Array.isArray(parsed)).toBe(true); + expect(parsed.some((g: any) => g.name === "all")).toBe(true); + }); + }); + + test("group list shows table output", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const result = await backend.runCLICommand( + ["group", "list"], + tempDir + ); + + expect(result.code).toEqual(0); + expect(result.stdout).toContain("Name"); + expect(result.stdout).toContain("Summary"); + expect(result.stdout).toContain("Members"); + }); + }); + + test("group create + get + delete lifecycle", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const groupName = `cli_test_${Date.now()}`; + + // Create + const createResult = await backend.runCLICommand( + ["group", "create", groupName, "--summary", "CLI test group"], + tempDir + ); + expect(createResult.code).toEqual(0); + expect(createResult.stdout).toContain("created"); + + // Get + const getResult = await backend.runCLICommand( + ["group", "get", groupName], + tempDir + ); + expect(getResult.code).toEqual(0); + expect(getResult.stdout).toContain(groupName); + expect(getResult.stdout).toContain("CLI test group"); + + // Get --json + const getJsonResult = await backend.runCLICommand( + ["group", "get", groupName, "--json"], + tempDir + ); + expect(getJsonResult.code).toEqual(0); + const parsed = JSON.parse(getJsonResult.stdout); + expect(parsed.name).toBe(groupName); + + // Delete + const deleteResult = await backend.runCLICommand( + ["group", "delete", groupName], + tempDir + ); + expect(deleteResult.code).toEqual(0); + expect(deleteResult.stdout).toContain("deleted"); + }); + }); + + test("group add-user and remove-user", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const groupName = `cli_member_test_${Date.now()}`; + + await backend.runCLICommand(["group", "create", groupName], tempDir); + + // Add user + const addResult = await backend.runCLICommand( + ["group", "add-user", groupName, "admin@windmill.dev"], + tempDir + ); + expect(addResult.code).toEqual(0); + expect(addResult.stdout).toContain("added"); + + // Remove user + const removeResult = await backend.runCLICommand( + ["group", "remove-user", groupName, "admin@windmill.dev"], + tempDir + ); + expect(removeResult.code).toEqual(0); + expect(removeResult.stdout).toContain("removed"); + + // Cleanup + await backend.runCLICommand(["group", "delete", groupName], tempDir); + }); + }); + + test("default action (wmill group) lists groups", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const result = await backend.runCLICommand(["group", "--json"], tempDir); + + expect(result.code).toEqual(0); + const parsed = JSON.parse(result.stdout); + expect(Array.isArray(parsed)).toBe(true); + }); + }); + + test("group --help shows all subcommands", async () => { + await withTestBackend(async (backend, tempDir) => { + const result = await backend.runCLICommand(["group", "--help"], tempDir); + const output = result.stdout + result.stderr; + expect(output).toContain("list"); + expect(output).toContain("get"); + expect(output).toContain("create"); + expect(output).toContain("delete"); + expect(output).toContain("add-user"); + expect(output).toContain("remove-user"); + }); + }); +}); diff --git a/cli/test/init_template_unit.test.ts b/cli/test/init_template_unit.test.ts new file mode 100644 index 0000000000..efb6d09891 --- /dev/null +++ b/cli/test/init_template_unit.test.ts @@ -0,0 +1,244 @@ +/** + * Unit tests for wmill.yaml template generation, config reference, and JSON Schema. + */ + +import { expect, test, describe } from "bun:test"; +import { parse } from "yaml"; +import Ajv from "ajv"; +import { + generateCommentedTemplate, + generateJsonSchema, + formatConfigReference, + formatConfigReferenceJson, + CONFIG_REFERENCE, +} from "../src/commands/init/template.ts"; + +// ============================================================================= +// generateCommentedTemplate +// ============================================================================= + +describe("generateCommentedTemplate", () => { + test("produces valid YAML that parses without errors", () => { + const yaml = generateCommentedTemplate("main"); + const config = parse(yaml); + expect(config).toBeDefined(); + expect(typeof config).toBe("object"); + }); + + test("uses provided branch name in gitBranches", () => { + const config = parse(generateCommentedTemplate("my-feature")); + expect(config.gitBranches["my-feature"]).toBeDefined(); + expect(config.gitBranches["my-feature"].overrides).toEqual({}); + }); + + test("defaults to 'main' when no branch name given", () => { + const config = parse(generateCommentedTemplate()); + expect(config.gitBranches["main"]).toBeDefined(); + }); + + test("quotes branch names with YAML-special characters", () => { + const specialBranches = ["fix: something", "feat/my branch", "release#1"]; + for (const branch of specialBranches) { + const yaml = generateCommentedTemplate(branch); + const config = parse(yaml); + expect(config.gitBranches[branch]).toBeDefined(); + } + }); + + test("contains yaml-language-server schema directive", () => { + const yaml = generateCommentedTemplate("main"); + expect(yaml.startsWith("# yaml-language-server: $schema=wmill.schema.json")).toBe(true); + }); + + test("includes all non-commented CONFIG_REFERENCE entries as active YAML keys", () => { + const config = parse(generateCommentedTemplate("main")); + for (const opt of CONFIG_REFERENCE) { + if (!opt.commented) { + expect(config).toHaveProperty(opt.name); + } + } + }); + + test("does not include commented entries as active YAML keys", () => { + const config = parse(generateCommentedTemplate("main")); + for (const opt of CONFIG_REFERENCE) { + if (opt.commented && opt.name !== "environments") { + expect(config[opt.name]).toBeUndefined(); + } + } + }); + + test("default values match expected defaults", () => { + const config = parse(generateCommentedTemplate("main")); + expect(config.defaultTs).toBe("bun"); + expect(config.skipSecrets).toBe(true); + expect(config.nonDottedPaths).toBe(true); + expect(config.codebases).toEqual([]); + expect(config.excludes).toEqual([]); + expect(config.includes).toEqual(["f/**"]); + }); +}); + +// ============================================================================= +// generateJsonSchema +// ============================================================================= + +describe("generateJsonSchema", () => { + const schema = generateJsonSchema(); + + test("is a valid JSON Schema draft-07", () => { + expect(schema.$schema).toBe("http://json-schema.org/draft-07/schema#"); + expect(schema.type).toBe("object"); + expect(schema.properties).toBeDefined(); + }); + + test("validates the generated YAML template", () => { + const config = parse(generateCommentedTemplate("main")); + const ajv = new Ajv({ allErrors: true }); + const validate = ajv.compile(schema); + expect(validate(config)).toBe(true); + }); + + test("rejects unknown keys", () => { + const ajv = new Ajv({ allErrors: true }); + const validate = ajv.compile(schema); + expect(validate({ unknownOption: true })).toBe(false); + }); + + test("rejects invalid enum values", () => { + const ajv = new Ajv({ allErrors: true }); + const validate = ajv.compile(schema); + expect(validate({ defaultTs: "python" })).toBe(false); + }); + + test("rejects wrong types", () => { + const ajv = new Ajv({ allErrors: true }); + const validate = ajv.compile(schema); + expect(validate({ skipSecrets: "yes" })).toBe(false); + }); + + test("includes codebases array schema with item properties", () => { + expect(schema.properties.codebases.type).toBe("array"); + expect(schema.properties.codebases.items.properties.relative_path).toBeDefined(); + expect(schema.properties.codebases.items.required).toContain("relative_path"); + }); + + test("includes gitBranches with branch config schema", () => { + const branchSchema = schema.properties.gitBranches.additionalProperties; + expect(branchSchema.properties.baseUrl).toBeDefined(); + expect(branchSchema.properties.workspaceId).toBeDefined(); + expect(branchSchema.properties.specificItems).toBeDefined(); + expect(branchSchema.properties.specificItems.properties.variables).toBeDefined(); + }); + + test("includes environments as alias for gitBranches", () => { + expect(schema.properties.environments).toBeDefined(); + expect(schema.properties.environments.additionalProperties).toEqual( + schema.properties.gitBranches.additionalProperties + ); + }); + + test("does not contain template-only keys in schema output", () => { + const templateKeys = ["section", "sectionNote", "commented", "templateValue", "example", "inlineComment", "groupNote"]; + const json = JSON.stringify(schema); + for (const key of templateKeys) { + expect(json).not.toContain(`"${key}"`); + } + }); +}); + +// ============================================================================= +// formatConfigReference +// ============================================================================= + +describe("formatConfigReference", () => { + const output = formatConfigReference(); + + test("includes header row", () => { + expect(output).toContain("OPTION"); + expect(output).toContain("DESCRIPTION"); + expect(output).toContain("DEFAULT"); + }); + + test("includes all top-level CONFIG_REFERENCE entries", () => { + for (const opt of CONFIG_REFERENCE) { + expect(output).toContain(opt.name); + } + }); + + test("auto-expands codebases sub-fields", () => { + expect(output).toContain("codebases[].relative_path"); + expect(output).toContain("codebases[].format"); + expect(output).toContain("codebases[].external"); + }); + + test("auto-expands gitBranches sub-fields", () => { + expect(output).toContain("gitBranches..baseUrl"); + expect(output).toContain("gitBranches..workspaceId"); + expect(output).toContain("gitBranches..specificItems.variables"); + }); + + test("auto-expands commonSpecificItems sub-fields", () => { + expect(output).toContain("gitBranches.commonSpecificItems.variables"); + expect(output).toContain("gitBranches.commonSpecificItems.settings"); + }); +}); + +// ============================================================================= +// formatConfigReferenceJson +// ============================================================================= + +describe("formatConfigReferenceJson", () => { + test("produces valid JSON", () => { + const parsed = JSON.parse(formatConfigReferenceJson()); + expect(Array.isArray(parsed)).toBe(true); + expect(parsed.length).toBe(CONFIG_REFERENCE.length); + }); + + test("each entry has name, type, default, description", () => { + const parsed = JSON.parse(formatConfigReferenceJson()); + for (const entry of parsed) { + expect(entry).toHaveProperty("name"); + expect(entry).toHaveProperty("type"); + expect(entry).toHaveProperty("default"); + expect(entry).toHaveProperty("description"); + } + }); + + test("does not contain template-only keys", () => { + const parsed = JSON.parse(formatConfigReferenceJson()); + const templateKeys = ["section", "sectionNote", "commented", "templateValue", "example", "inlineComment", "groupNote"]; + for (const entry of parsed) { + for (const key of templateKeys) { + expect(entry).not.toHaveProperty(key); + } + } + }); +}); + +// ============================================================================= +// CONFIG_REFERENCE integrity +// ============================================================================= + +describe("CONFIG_REFERENCE integrity", () => { + test("all entries have required fields", () => { + for (const opt of CONFIG_REFERENCE) { + expect(opt.name).toBeTruthy(); + expect(opt.type).toBeTruthy(); + expect(opt.description).toBeTruthy(); + expect(opt.default).toBeDefined(); + } + }); + + test("no duplicate names", () => { + const names = CONFIG_REFERENCE.map((o) => o.name); + expect(new Set(names).size).toBe(names.length); + }); + + test("type field uses valid JSON Schema types", () => { + const validTypes = new Set(["boolean", "string", "integer", "number", "array", "object"]); + for (const opt of CONFIG_REFERENCE) { + expect(validTypes.has(opt.type)).toBe(true); + } + }); +}); diff --git a/cli/test/inline_scripts_failure_preprocessor.test.ts b/cli/test/inline_scripts_failure_preprocessor_unit.test.ts similarity index 83% rename from cli/test/inline_scripts_failure_preprocessor.test.ts rename to cli/test/inline_scripts_failure_preprocessor_unit.test.ts index 1a9150a257..78af37a02e 100644 --- a/cli/test/inline_scripts_failure_preprocessor.test.ts +++ b/cli/test/inline_scripts_failure_preprocessor_unit.test.ts @@ -496,3 +496,81 @@ describe("extractCurrentMapping for failure_module / preprocessor_module", () => expect(mapping["failure"]).toBeUndefined(); }); }); + +// --------------------------------------------------------------------------- +// extractInlineScripts with mapping — path preservation +// --------------------------------------------------------------------------- + +describe("extractInlineScripts with mapping preserves file paths", () => { + test("uses mapped path instead of assigner-generated path", () => { + const mod = makeRawscriptModule("a", "console.log('hi')", "bun"); + mod.summary = "Get Users Data"; + + const mapping = { a: "get_users.ts" }; + const scripts = extractInlineScripts([mod], mapping, "/", "bun"); + + const contentScript = scripts.find((s) => !s.is_lock); + expect(contentScript!.path).toBe("get_users.ts"); + // Module content should reference the mapped path + expect(mod.value.content).toBe("!inline get_users.ts"); + }); + + test("falls through to assigner when module ID not in mapping", () => { + const mod = makeRawscriptModule("a", "console.log('hi')", "bun"); + mod.summary = "Get Users Data"; + + const mapping = { other_id: "other.ts" }; + const scripts = extractInlineScripts([mod], mapping, "/", "bun"); + + const contentScript = scripts.find((s) => !s.is_lock); + // Should use assigner path based on summary, not mapped + expect(contentScript!.path).toContain("get_users_data"); + }); + + test("mapped modules and unmapped modules coexist", () => { + const modA = makeRawscriptModule("a", "code_a", "bun"); + modA.summary = "Step A"; + const modB = makeRawscriptModule("b", "code_b", "bun"); + modB.summary = "Step B"; + + const mapping = { a: "my_custom_name.ts" }; // only a is mapped + const scripts = extractInlineScripts([modA, modB], mapping, "/", "bun"); + + const paths = scripts.filter((s) => !s.is_lock).map((s) => s.path); + expect(paths[0]).toBe("my_custom_name.ts"); + expect(paths[1]).toContain("step_b"); // assigner-generated from summary + }); + + test("lock path is derived from mapped content path", () => { + const mod = makeRawscriptModule("a", "code", "bun", "lock-content"); + mod.summary = "Get Users Data"; + + const mapping = { a: "get_users.ts" }; + const scripts = extractInlineScripts([mod], mapping, "/", "bun"); + + const lockScript = scripts.find((s) => s.is_lock); + expect(lockScript!.path).toBe("get_users.lock"); + expect((mod.value as any).lock).toBe("!inline get_users.lock"); + }); + + test("lock path uses assigner basePath when no mapping", () => { + const mod = makeRawscriptModule("a", "code", "bun", "lock-content"); + mod.summary = "Get Users Data"; + + const scripts = extractInlineScripts([mod], {}, "/", "bun"); + + const lockScript = scripts.find((s) => s.is_lock); + expect(lockScript!.path).toContain("get_users_data"); + expect(lockScript!.path).toEndWith(".lock"); + }); + + test("lock path handles dotted content paths correctly", () => { + const mod = makeRawscriptModule("a", "code", "bun", "lock-content"); + + const mapping = { a: "my.inline_script.ts" }; + const scripts = extractInlineScripts([mod], mapping, "/", "bun"); + + const lockScript = scripts.find((s) => s.is_lock); + expect(lockScript!.path).toBe("my.inline_script.lock"); + }); +}); diff --git a/cli/test/job_commands.test.ts b/cli/test/job_commands.test.ts new file mode 100644 index 0000000000..0249c3bf6b --- /dev/null +++ b/cli/test/job_commands.test.ts @@ -0,0 +1,348 @@ +/** + * Integration tests for `wmill job` commands: + * list, get, result, logs, cancel + */ + +import { expect, test, describe } from "bun:test"; +import { withTestBackend } from "./test_backend.ts"; +import { + setupWorkspaceProfile, + createRemoteScript, + createRemoteFlow, + createRemoteMultiStepFlow, + createRemoteFailingFlow, + runRemoteScript, + runRemoteFlow, + waitForJob, +} from "./new_commands_helpers.ts"; + +describe("job command", () => { + test("job list returns valid JSON", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const scriptPath = `f/test/job_test_${uniqueId}`; + await createRemoteScript(backend, scriptPath); + const jobId = await runRemoteScript(backend, scriptPath); + await waitForJob(backend, jobId); + + const result = await backend.runCLICommand( + ["job", "list", "--json"], + tempDir + ); + + expect(result.code).toEqual(0); + const parsed = JSON.parse(result.stdout); + expect(Array.isArray(parsed)).toBe(true); + expect(parsed.some((j: any) => j.id === jobId)).toBe(true); + }); + }); + + test("job list shows table output", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const scriptPath = `f/test/job_table_${uniqueId}`; + await createRemoteScript(backend, scriptPath); + const jobId = await runRemoteScript(backend, scriptPath); + await waitForJob(backend, jobId); + + const result = await backend.runCLICommand( + ["job", "list"], + tempDir + ); + + expect(result.code).toEqual(0); + expect(result.stdout).toContain("ID"); + expect(result.stdout).toContain("Status"); + expect(result.stdout).toContain(jobId); + }); + }); + + test("job list --script-path filters correctly", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const scriptPath = `f/test/filter_test_${uniqueId}`; + await createRemoteScript(backend, scriptPath); + const jobId = await runRemoteScript(backend, scriptPath); + await waitForJob(backend, jobId); + + const result = await backend.runCLICommand( + ["job", "list", "--json", "--script-path", scriptPath], + tempDir + ); + + expect(result.code).toEqual(0); + const parsed = JSON.parse(result.stdout); + expect(parsed.every((j: any) => j.script_path === scriptPath)).toBe(true); + }); + }); + + test("job get returns job details", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const scriptPath = `f/test/job_get_${uniqueId}`; + await createRemoteScript(backend, scriptPath); + const jobId = await runRemoteScript(backend, scriptPath); + await waitForJob(backend, jobId); + + const result = await backend.runCLICommand( + ["job", "get", jobId], + tempDir + ); + + expect(result.code).toEqual(0); + expect(result.stdout).toContain("ID:"); + expect(result.stdout).toContain(jobId); + expect(result.stdout).toContain("Status:"); + expect(result.stdout).toContain("success"); + }); + }); + + test("job get --json returns valid JSON", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const scriptPath = `f/test/job_get_json_${uniqueId}`; + await createRemoteScript(backend, scriptPath); + const jobId = await runRemoteScript(backend, scriptPath); + await waitForJob(backend, jobId); + + const result = await backend.runCLICommand( + ["job", "get", jobId, "--json"], + tempDir + ); + + expect(result.code).toEqual(0); + const parsed = JSON.parse(result.stdout); + expect(parsed.id).toBe(jobId); + expect(parsed.success).toBe(true); + }); + }); + + test("job result returns job result as JSON", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const scriptPath = `f/test/job_result_${uniqueId}`; + await createRemoteScript(backend, scriptPath); + const jobId = await runRemoteScript(backend, scriptPath); + await waitForJob(backend, jobId); + + const result = await backend.runCLICommand( + ["job", "result", jobId], + tempDir + ); + + expect(result.code).toEqual(0); + // Result may be in stdout or combined output + const output = result.stdout.trim(); + expect(output.length).toBeGreaterThan(0); + const parsed = JSON.parse(output); + expect(parsed).toBe("hello"); + }); + }); + + test("job logs returns job logs", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const scriptPath = `f/test/job_logs_${uniqueId}`; + await createRemoteScript(backend, scriptPath); + const jobId = await runRemoteScript(backend, scriptPath); + await waitForJob(backend, jobId); + + const result = await backend.runCLICommand( + ["job", "logs", jobId], + tempDir + ); + + expect(result.code).toEqual(0); + expect(result.stdout.length).toBeGreaterThan(0); + }); + }); + + test("job logs for flow job aggregates step logs", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const flowPath = `f/test/flow_logs_${uniqueId}`; + await createRemoteMultiStepFlow(backend, flowPath); + const jobId = await runRemoteFlow(backend, flowPath); + await waitForJob(backend, jobId); + + const result = await backend.runCLICommand( + ["job", "logs", jobId], + tempDir + ); + + expect(result.code).toEqual(0); + // Should show labeled step headers instead of "no direct logs" + expect(result.stdout).toContain("======"); + expect(result.stdout).toContain("a: Generate data"); + }); + }); + + test("default action (wmill job) lists jobs", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const result = await backend.runCLICommand( + ["job", "--json"], + tempDir + ); + + expect(result.code).toEqual(0); + const parsed = JSON.parse(result.stdout); + expect(Array.isArray(parsed)).toBe(true); + }); + }); + + test("job --help shows all subcommands", async () => { + await withTestBackend(async (backend, tempDir) => { + const result = await backend.runCLICommand(["job", "--help"], tempDir); + const output = result.stdout + result.stderr; + expect(output).toContain("list"); + expect(output).toContain("get"); + expect(output).toContain("result"); + expect(output).toContain("logs"); + expect(output).toContain("cancel"); + expect(output).toContain("--failed"); + expect(output).toContain("--running"); + expect(output).toContain("--parent"); + expect(output).toContain("--is-flow-step"); + }); + }); + + test("job get for flow shows hierarchical step tree", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const flowPath = `f/test/flow_get_${uniqueId}`; + await createRemoteMultiStepFlow(backend, flowPath); + const jobId = await runRemoteFlow(backend, flowPath); + await waitForJob(backend, jobId); + + const result = await backend.runCLICommand( + ["job", "get", jobId], + tempDir + ); + + expect(result.code).toEqual(0); + expect(result.stdout).toContain("Steps:"); + // Should show step IDs from the flow definition + expect(result.stdout).toContain("a"); + expect(result.stdout).toContain("b"); + // Should show status icons (✓ for success) + expect(result.stdout).toContain("✓"); + }); + }); + + test("job get --json for flow includes flow_status", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const flowPath = `f/test/flow_json_${uniqueId}`; + await createRemoteMultiStepFlow(backend, flowPath); + const jobId = await runRemoteFlow(backend, flowPath); + await waitForJob(backend, jobId); + + const result = await backend.runCLICommand( + ["job", "get", jobId, "--json"], + tempDir + ); + + expect(result.code).toEqual(0); + const parsed = JSON.parse(result.stdout); + expect(parsed.flow_status).toBeDefined(); + expect(parsed.flow_status.modules).toBeDefined(); + expect(parsed.flow_status.modules.length).toBe(2); + }); + }); + + test("job list --parent shows sub-jobs of a flow", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const flowPath = `f/test/flow_parent_${uniqueId}`; + await createRemoteMultiStepFlow(backend, flowPath); + const jobId = await runRemoteFlow(backend, flowPath); + await waitForJob(backend, jobId); + + const result = await backend.runCLICommand( + ["job", "list", "--json", "--parent", jobId], + tempDir + ); + + expect(result.code).toEqual(0); + const parsed = JSON.parse(result.stdout); + expect(Array.isArray(parsed)).toBe(true); + // A 2-step flow should have at least 2 sub-jobs + expect(parsed.length).toBeGreaterThanOrEqual(2); + // All sub-jobs should reference the parent flow + expect(parsed.every((j: any) => j.parent_job === jobId)).toBe(true); + }); + }); + + test("job list --all includes sub-jobs", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const flowPath = `f/test/flow_all_${uniqueId}`; + await createRemoteMultiStepFlow(backend, flowPath); + const jobId = await runRemoteFlow(backend, flowPath); + await waitForJob(backend, jobId); + + const result = await backend.runCLICommand( + ["job", "list", "--json", "--all"], + tempDir + ); + + expect(result.code).toEqual(0); + const parsed = JSON.parse(result.stdout); + // Should contain both the parent flow and its sub-jobs + const parentJob = parsed.find((j: any) => j.id === jobId); + const subJobs = parsed.filter((j: any) => j.parent_job === jobId); + expect(parentJob).toBeDefined(); + expect(subJobs.length).toBeGreaterThanOrEqual(2); + }); + }); + + test("job get for failed flow shows failure status", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const flowPath = `f/test/flow_fail_${uniqueId}`; + await createRemoteFailingFlow(backend, flowPath); + const jobId = await runRemoteFlow(backend, flowPath); + await waitForJob(backend, jobId); + + const result = await backend.runCLICommand( + ["job", "get", jobId], + tempDir + ); + + expect(result.code).toEqual(0); + expect(result.stdout).toContain("failure"); + expect(result.stdout).toContain("Steps:"); + // Step a should succeed, step b should fail + expect(result.stdout).toContain("✓"); + expect(result.stdout).toContain("✗"); + }); + }); +}); diff --git a/cli/test/lint_command.test.ts b/cli/test/lint_command_unit.test.ts similarity index 100% rename from cli/test/lint_command.test.ts rename to cli/test/lint_command_unit.test.ts diff --git a/cli/test/lint_locks.test.ts b/cli/test/lint_locks_unit.test.ts similarity index 100% rename from cli/test/lint_locks.test.ts rename to cli/test/lint_locks_unit.test.ts diff --git a/cli/test/list_get_new_commands.test.ts b/cli/test/list_get_new_commands.test.ts index 8df67cab9b..df3d109630 100644 --- a/cli/test/list_get_new_commands.test.ts +++ b/cli/test/list_get_new_commands.test.ts @@ -438,6 +438,38 @@ describe("new command", () => { }); }); + test("flow new respects nonDottedPaths: true", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + await writeFile( + join(tempDir, "wmill.yaml"), + `defaultTs: bun\nnonDottedPaths: true\nincludes:\n - "**"\nexcludes: []\n`, + "utf-8" + ); + await mkdir(join(tempDir, "f", "test"), { recursive: true }); + + const result = await backend.runCLICommand( + ["flow", "new", "f/test/nondot_flow", "--summary", "Non-dotted flow"], + tempDir + ); + + expect(result.code).toEqual(0); + + // Should use __flow suffix, not .flow + const flowYamlStat = await stat( + join(tempDir, "f/test/nondot_flow__flow/flow.yaml") + ); + expect(flowYamlStat.isFile()).toBe(true); + + const flowContent = await readFile( + join(tempDir, "f/test/nondot_flow__flow/flow.yaml"), + "utf-8" + ); + expect(flowContent).toContain("Non-dotted flow"); + }); + }); + test("flow bootstrap still works as alias", async () => { await withTestBackend(async (backend, tempDir) => { await setupWorkspaceProfile(backend); diff --git a/cli/test/lock_cache.test.ts b/cli/test/lock_cache_unit.test.ts similarity index 100% rename from cli/test/lock_cache.test.ts rename to cli/test/lock_cache_unit.test.ts diff --git a/cli/test/new_commands_helpers.ts b/cli/test/new_commands_helpers.ts new file mode 100644 index 0000000000..4702ca5eb6 --- /dev/null +++ b/cli/test/new_commands_helpers.ts @@ -0,0 +1,325 @@ +/** + * Shared helpers for new CLI command tests. + */ + +import { expect } from "bun:test"; +import { type TestBackend } from "./test_backend.ts"; +import { addWorkspace } from "../workspace.ts"; + +export async function setupWorkspaceProfile(backend: TestBackend): Promise { + await addWorkspace( + { + remote: backend.baseUrl, + workspaceId: backend.workspace, + name: "localhost_test", + token: backend.token!, + }, + { force: true, configDir: backend.testConfigDir } + ); +} + +export async function ensureFolder(backend: TestBackend, name: string): Promise { + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/folders/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name }), + } + ); + await resp.text(); +} + +export async function createRemoteScript( + backend: TestBackend, + scriptPath: string, + content: string = 'export async function main() { return "hello"; }' +): Promise { + const parts = scriptPath.split("/"); + if (parts[0] === "f" && parts.length > 2) { + await ensureFolder(backend, parts[1]); + } + + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: scriptPath, + content, + language: "bun", + summary: "Test script", + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + }), + } + ); + expect(resp.status).toBeLessThan(300); + await resp.text(); +} + +export async function runRemoteScript( + backend: TestBackend, + scriptPath: string, + retries: number = 10 +): Promise { + for (let i = 0; i < retries; i++) { + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/jobs/run/p/${scriptPath}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + } + ); + if (resp.status < 300) { + return (await resp.text()).replace(/"/g, ""); + } + await resp.text(); + if (i < retries - 1) { + await new Promise((r) => setTimeout(r, 1000)); + } + } + throw new Error(`Failed to run script ${scriptPath} after ${retries} retries`); +} + +export async function waitForJob( + backend: TestBackend, + jobId: string, + timeoutMs: number = 15000 +): Promise { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/jobs_u/completed/get/${jobId}` + ); + if (resp.ok) return; + await new Promise((r) => setTimeout(r, 500)); + } + throw new Error(`Job ${jobId} did not complete within ${timeoutMs}ms`); +} + +export async function createRemoteFlow( + backend: TestBackend, + flowPath: string +): Promise { + const parts = flowPath.split("/"); + if (parts[0] === "f" && parts.length > 2) { + await ensureFolder(backend, parts[1]); + } + + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/flows/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: flowPath, + summary: "Test flow", + description: "A test flow", + value: { + modules: [ + { + id: "a", + value: { + type: "rawscript", + content: 'export async function main() { return "flow done"; }', + language: "bun", + input_transforms: {}, + }, + }, + ], + }, + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + }), + } + ); + expect(resp.status).toBeLessThan(300); + await resp.text(); +} + +export async function runRemoteFlow( + backend: TestBackend, + flowPath: string, + retries: number = 10 +): Promise { + for (let i = 0; i < retries; i++) { + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/jobs/run/f/${flowPath}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + } + ); + if (resp.status < 300) { + return (await resp.text()).replace(/"/g, ""); + } + await new Promise((r) => setTimeout(r, 500)); + } + throw new Error(`Failed to run flow ${flowPath} after ${retries} retries`); +} + +/** + * Create a multi-step flow with 2 steps (a prints, b returns result). + * Useful for testing hierarchical job get and aggregated logs. + */ +export async function createRemoteMultiStepFlow( + backend: TestBackend, + flowPath: string +): Promise { + const parts = flowPath.split("/"); + if (parts[0] === "f" && parts.length > 2) { + await ensureFolder(backend, parts[1]); + } + + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/flows/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: flowPath, + summary: "Multi-step test flow", + description: "A flow with two steps for testing", + value: { + modules: [ + { + id: "a", + summary: "Generate data", + value: { + type: "rawscript", + content: + 'export async function main() { console.log("step a running"); return { value: 42 }; }', + language: "bun", + input_transforms: {}, + }, + }, + { + id: "b", + summary: "Process data", + value: { + type: "rawscript", + content: + 'export async function main(data: any) { console.log("step b running"); return "done"; }', + language: "bun", + input_transforms: { + data: { type: "javascript", expr: "results.a" }, + }, + }, + }, + ], + }, + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + }), + } + ); + expect(resp.status).toBeLessThan(300); + await resp.text(); +} + +/** + * Create a flow where step b throws an error. + * Useful for testing failure handling. + */ +export async function createRemoteFailingFlow( + backend: TestBackend, + flowPath: string +): Promise { + const parts = flowPath.split("/"); + if (parts[0] === "f" && parts.length > 2) { + await ensureFolder(backend, parts[1]); + } + + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/flows/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: flowPath, + summary: "Failing test flow", + description: "A flow where step b fails", + value: { + modules: [ + { + id: "a", + summary: "Succeeding step", + value: { + type: "rawscript", + content: + 'export async function main() { return "ok"; }', + language: "bun", + input_transforms: {}, + }, + }, + { + id: "b", + summary: "Failing step", + value: { + type: "rawscript", + content: + 'export async function main() { throw new Error("simulated failure"); }', + language: "bun", + input_transforms: {}, + }, + }, + ], + }, + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + }), + } + ); + expect(resp.status).toBeLessThan(300); + await resp.text(); +} + +export async function createRemoteSchedule( + backend: TestBackend, + schedulePath: string, + scriptPath: string +): Promise { + const parts = schedulePath.split("/"); + if (parts[0] === "f" && parts.length > 2) { + await ensureFolder(backend, parts[1]); + } + + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/schedules/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: schedulePath, + schedule: "0 0 */6 * * *", + timezone: "Etc/UTC", + script_path: scriptPath, + is_flow: false, + args: {}, + enabled: false, + }), + } + ); + expect(resp.status).toBeLessThan(300); + await resp.text(); +} diff --git a/cli/test/replace_path_scripts.test.ts b/cli/test/replace_path_scripts_unit.test.ts similarity index 100% rename from cli/test/replace_path_scripts.test.ts rename to cli/test/replace_path_scripts_unit.test.ts diff --git a/cli/test/resource_folders_unit.test.ts b/cli/test/resource_folders_unit.test.ts index 80732ebc1c..0d07812124 100644 --- a/cli/test/resource_folders_unit.test.ts +++ b/cli/test/resource_folders_unit.test.ts @@ -31,6 +31,8 @@ import { isAppMetadataFile, isRawAppMetadataFile, isRawAppFolderMetadataFile, + isAppFolderMetadataFile, + isFlowFolderMetadataFile, getDeleteSuffix, transformJsonPathToDir, isModuleEntryPoint, @@ -495,6 +497,38 @@ describe("isRawAppFolderMetadataFile", () => { }); }); +describe("isAppFolderMetadataFile", () => { + test("detects app folder metadata file (dotted)", () => { + expect(isAppFolderMetadataFile("f/common/landing.app/app.yaml")).toBe(true); + expect(isAppFolderMetadataFile("f/common/landing.app/app.json")).toBe(true); + }); + + test("rejects inline script files inside app folder", () => { + expect(isAppFolderMetadataFile("f/common/landing.app/eval_of_e.inline_script.frontend.js")).toBe(false); + expect(isAppFolderMetadataFile("f/common/landing.app/button1.inline_script.bun.ts")).toBe(false); + }); + + test("rejects top-level app metadata files", () => { + expect(isAppFolderMetadataFile("f/common/landing.app.json")).toBe(false); + expect(isAppFolderMetadataFile("f/common/landing.app.yaml")).toBe(false); + }); +}); + +describe("isFlowFolderMetadataFile", () => { + test("detects flow folder metadata file (dotted)", () => { + expect(isFlowFolderMetadataFile("f/common/my_flow.flow/flow.yaml")).toBe(true); + expect(isFlowFolderMetadataFile("f/common/my_flow.flow/flow.json")).toBe(true); + }); + + test("rejects inline script files inside flow folder", () => { + expect(isFlowFolderMetadataFile("f/common/my_flow.flow/step_0.inline_script.ts")).toBe(false); + }); + + test("rejects top-level flow metadata files", () => { + expect(isFlowFolderMetadataFile("f/common/my_flow.flow.json")).toBe(false); + }); +}); + // ============================================================================= // Sync-related Path Functions // ============================================================================= diff --git a/cli/test/schedule_history_commands.test.ts b/cli/test/schedule_history_commands.test.ts new file mode 100644 index 0000000000..eca094a77d --- /dev/null +++ b/cli/test/schedule_history_commands.test.ts @@ -0,0 +1,207 @@ +/** + * Integration tests for: + * - `wmill schedule enable/disable` + * - `wmill script history` + * - `wmill flow history/show-version` + */ + +import { expect, test, describe } from "bun:test"; +import { withTestBackend } from "./test_backend.ts"; +import { + setupWorkspaceProfile, + createRemoteScript, + createRemoteFlow, + createRemoteSchedule, +} from "./new_commands_helpers.ts"; + +// ============================================================================= +// schedule enable/disable commands +// ============================================================================= + +describe("schedule enable/disable", () => { + test("schedule enable and disable toggle schedule state", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const scriptPath = `f/test/sched_script_${uniqueId}`; + const schedulePath = `f/test/sched_${uniqueId}`; + + await createRemoteScript(backend, scriptPath); + await createRemoteSchedule(backend, schedulePath, scriptPath); + + // Enable + const enableResult = await backend.runCLICommand( + ["schedule", "enable", schedulePath], + tempDir + ); + expect(enableResult.code).toEqual(0); + expect(enableResult.stdout).toContain("enabled"); + + // Verify enabled via get + const getResult1 = await backend.runCLICommand( + ["schedule", "get", schedulePath, "--json"], + tempDir + ); + expect(getResult1.code).toEqual(0); + const schedule1 = JSON.parse(getResult1.stdout); + expect(schedule1.enabled).toBe(true); + + // Disable + const disableResult = await backend.runCLICommand( + ["schedule", "disable", schedulePath], + tempDir + ); + expect(disableResult.code).toEqual(0); + expect(disableResult.stdout).toContain("disabled"); + + // Verify disabled via get + const getResult2 = await backend.runCLICommand( + ["schedule", "get", schedulePath, "--json"], + tempDir + ); + expect(getResult2.code).toEqual(0); + const schedule2 = JSON.parse(getResult2.stdout); + expect(schedule2.enabled).toBe(false); + }); + }); + + test("schedule enable/disable shows in help", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const result = await backend.runCLICommand( + ["schedule", "--help"], + tempDir + ); + + expect(result.code).toEqual(0); + const output = result.stdout + result.stderr; + expect(output).toContain("enable"); + expect(output).toContain("disable"); + }); + }); +}); + +// ============================================================================= +// script history command +// ============================================================================= + +describe("script history", () => { + test("script history returns version list", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const scriptPath = `f/test/history_script_${uniqueId}`; + await createRemoteScript(backend, scriptPath); + + const result = await backend.runCLICommand( + ["script", "history", scriptPath, "--json"], + tempDir + ); + + expect(result.code).toEqual(0); + const parsed = JSON.parse(result.stdout); + expect(Array.isArray(parsed)).toBe(true); + expect(parsed.length).toBeGreaterThanOrEqual(1); + expect(parsed[0]).toHaveProperty("script_hash"); + }); + }); + + test("script history shows table output", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const scriptPath = `f/test/history_table_${uniqueId}`; + await createRemoteScript(backend, scriptPath); + + const result = await backend.runCLICommand( + ["script", "history", scriptPath], + tempDir + ); + + expect(result.code).toEqual(0); + expect(result.stdout).toContain("Hash"); + expect(result.stdout).toContain("Deployment Message"); + }); + }); +}); + +// ============================================================================= +// flow history and show-version commands +// ============================================================================= + +describe("flow history", () => { + test("flow history returns version list", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const flowPath = `f/test/history_flow_${uniqueId}`; + await createRemoteFlow(backend, flowPath); + + const result = await backend.runCLICommand( + ["flow", "history", flowPath, "--json"], + tempDir + ); + + expect(result.code).toEqual(0); + const parsed = JSON.parse(result.stdout); + expect(Array.isArray(parsed)).toBe(true); + expect(parsed.length).toBeGreaterThanOrEqual(1); + expect(parsed[0]).toHaveProperty("id"); + expect(parsed[0]).toHaveProperty("created_at"); + }); + }); + + test("flow history shows table output", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const flowPath = `f/test/history_table_flow_${uniqueId}`; + await createRemoteFlow(backend, flowPath); + + const result = await backend.runCLICommand( + ["flow", "history", flowPath], + tempDir + ); + + expect(result.code).toEqual(0); + expect(result.stdout).toContain("Version"); + expect(result.stdout).toContain("Created At"); + }); + }); + + test("flow show-version returns specific version", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const flowPath = `f/test/show_ver_flow_${uniqueId}`; + await createRemoteFlow(backend, flowPath); + + const histResult = await backend.runCLICommand( + ["flow", "history", flowPath, "--json"], + tempDir + ); + expect(histResult.code).toEqual(0); + const versions = JSON.parse(histResult.stdout); + expect(versions.length).toBeGreaterThanOrEqual(1); + + const versionId = String(versions[0].id); + + const showResult = await backend.runCLICommand( + ["flow", "show-version", flowPath, versionId, "--json"], + tempDir + ); + + expect(showResult.code).toEqual(0); + const flow = JSON.parse(showResult.stdout); + expect(flow.path).toBe(flowPath); + expect(flow.value).toBeDefined(); + }); + }); +}); diff --git a/cli/test/script_modules.test.ts b/cli/test/script_modules_unit.test.ts similarity index 100% rename from cli/test/script_modules.test.ts rename to cli/test/script_modules_unit.test.ts diff --git a/cli/test/setup.ts b/cli/test/setup.ts index 7eecd8bf2d..7f27240220 100644 --- a/cli/test/setup.ts +++ b/cli/test/setup.ts @@ -1,14 +1,22 @@ /** * Global test setup — preloaded before all test files. * + * When UNIT_ONLY=1, skips all backend setup (cargo build, database, etc.) + * so that unit tests can run instantly without any external dependencies. + * + * Otherwise: * 1. Builds the backend binary so `cargo run` starts instantly. * 2. Starts a shared backend instance so integration tests don't * bear the startup cost inside their per-test timeout window. */ -import { resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { statSync } from "node:fs"; +if (process.env["UNIT_ONLY"]) { + // Nothing to do — unit tests don't need backend setup +} else { + +const { resolve } = await import("node:path"); +const { fileURLToPath } = await import("node:url"); +const { statSync } = await import("node:fs"); const __dirname = resolve(fileURLToPath(import.meta.url), ".."); @@ -69,10 +77,22 @@ console.log("Backend build complete."); // This avoids the first integration test timing out while the backend // creates its database, starts the process, and waits for the health check. if (process.env["DATABASE_URL"]) { - const { getTestBackend } = await import("./test_backend.ts"); + // Clean up any stale databases/processes from previous crashed test runs + const { cleanupStaleTestResources } = await import("./cargo_backend.ts"); + await cleanupStaleTestResources(); + + const { getTestBackend, cleanupTestBackend } = await import("./test_backend.ts"); console.log("Pre-starting test backend..."); await getTestBackend(); console.log("Test backend is ready for all tests."); + + // Register afterAll to do full async cleanup (kill processes + drop DB) + // when all tests complete. The synchronous "exit" handler alone can't + // drop databases or scan /proc for orphaned child processes. + const { afterAll } = await import("bun:test"); + afterAll(async () => { + await cleanupTestBackend(); + }); } // When TEST_CLI_RUNTIME=node, also build the npm package so tests @@ -92,3 +112,5 @@ if (process.env["TEST_CLI_RUNTIME"] === "node") { } console.log("npm package built — tests will use Node runtime."); } + +} diff --git a/cli/test/specific_items.test.ts b/cli/test/specific_items_unit.test.ts similarity index 100% rename from cli/test/specific_items.test.ts rename to cli/test/specific_items_unit.test.ts diff --git a/cli/test/standalone_commands.test.ts b/cli/test/standalone_commands.test.ts index 1d75680354..bb1e13cde8 100644 --- a/cli/test/standalone_commands.test.ts +++ b/cli/test/standalone_commands.test.ts @@ -139,8 +139,10 @@ describe("resource-type commands", () => { ); expect(result.code).toEqual(0); - // Table headers should be present - expect(result.stdout).toContain("Name"); + // When empty, shows helpful message; when populated, shows table with Name header + const hasTable = result.stdout.includes("Name"); + const hasEmptyMessage = result.stdout.includes("No custom resource types"); + expect(hasTable || hasEmptyMessage).toBe(true); }); }); @@ -303,6 +305,79 @@ describe("script run command", () => { expect(result.stdout).toContain(`run_result_${uniqueId}`); }); }); + + test("exits with code 1 when script fails", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const scriptPath = `f/test/fail_script_${uniqueId}`; + const scriptContent = `export async function main() { throw new Error("intentional failure"); }`; + + await createRemoteScript(backend, scriptPath, scriptContent); + + const result = await backend.runCLICommand( + ["script", "run", scriptPath, "--silent"], + tempDir + ); + + expect(result.code).toEqual(1); + }); + }); + + test("errors when required args are missing", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const scriptPath = `f/test/args_script_${uniqueId}`; + const scriptContent = `export async function main(name: string) { return name; }`; + + // Create script with an explicit schema that has required args + // (createRemoteScript defaults to empty schema, so we call the API directly) + const parts = scriptPath.split("/"); + if (parts[0] === "f" && parts.length > 2) { + await backend.apiRequest!( + `/api/w/${backend.workspace}/folders/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: parts[1] }), + } + ).catch(() => {}); + } + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: scriptPath, + content: scriptContent, + language: "bun", + summary: "Test script with required args", + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: { name: { type: "string" } }, + required: ["name"], + }, + }), + } + ); + expect(resp.status).toBeLessThan(300); + await resp.text(); + + const result = await backend.runCLICommand( + ["script", "run", scriptPath], + tempDir + ); + + expect(result.code).not.toEqual(0); + const output = result.stdout + result.stderr; + expect(output).toContain("Missing required arguments"); + }); + }); }); // ============================================================================= @@ -569,3 +644,82 @@ describe("user commands", () => { }); }); }); + +// ============================================================================= +// Script Push --message +// ============================================================================= + +describe("script push --message", () => { + test("push with --message flag succeeds", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const scriptPath = `f/test/msg_script_${uniqueId}`; + const scriptFile = join(tempDir, scriptPath + ".ts"); + const metaFile = join(tempDir, scriptPath + ".script.yaml"); + const deployMsg = `deploy_msg_${uniqueId}`; + + await mkdir(join(tempDir, "f", "test"), { recursive: true }); + await writeFile(scriptFile, 'export async function main() { return "v1"; }'); + await writeFile(metaFile, [ + "summary: test", + "description: ''", + "lock: ''", + "kind: script", + "schema:", + " $schema: https://json-schema.org/draft/2020-12/schema", + " type: object", + " properties: {}", + " required: []", + ].join("\n")); + + // Verify push with --message flag succeeds (doesn't error on unknown flag) + const pushResult = await backend.runCLICommand( + ["script", "push", scriptPath + ".ts", "--message", deployMsg], + tempDir + ); + expect(pushResult.code).toEqual(0); + expect(pushResult.stdout).toContain("pushed"); + + // Verify history returns at least one version + const histResult = await backend.runCLICommand( + ["script", "history", scriptPath, "--json"], + tempDir + ); + expect(histResult.code).toEqual(0); + const versions = JSON.parse(histResult.stdout); + expect(versions.length).toBeGreaterThan(0); + }); + }); +}); + +// ============================================================================= +// Variable Add + Get (encryption roundtrip) +// ============================================================================= + +describe("variable add encryption", () => { + test("variable add creates a retrievable secret variable", { timeout: 30000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const varPath = `f/test/secret_${uniqueId}`; + const secretValue = `secret_value_${uniqueId}`; + + const addResult = await backend.runCLICommand( + ["variable", "add", secretValue, varPath], + tempDir + ); + expect(addResult.code).toEqual(0); + + const getResult = await backend.runCLICommand( + ["variable", "get", varPath], + tempDir + ); + expect(getResult.code).toEqual(0); + expect(getResult.stdout).toContain(secretValue); + expect(getResult.stdout).toContain("true"); // is_secret + }); + }); +}); diff --git a/cli/test/tar_creation.test.ts b/cli/test/tar_creation_unit.test.ts similarity index 100% rename from cli/test/tar_creation.test.ts rename to cli/test/tar_creation_unit.test.ts diff --git a/cli/test/test_backend.ts b/cli/test/test_backend.ts index 34ef1f240f..25943e2b98 100644 --- a/cli/test/test_backend.ts +++ b/cli/test/test_backend.ts @@ -590,11 +590,16 @@ function registerCleanup() { cleanupRegistered = true; process.on("exit", () => { if (globalBackend) { - // Synchronous kill — can't await in exit handler - try { - (globalBackend as any).backend?.process?.kill(); - } catch { - // Best effort + // Synchronous kill — can't await in exit handler. + // Kill the direct child (cargo); any orphaned windmill child processes + // will be cleaned up by cleanupStaleTestResources() on next startup. + const pid = (globalBackend as any).backend?.process?.pid; + if (pid) { + try { + process.kill(pid, "SIGKILL"); + } catch { + // Best effort — process may already be dead + } } } }); diff --git a/cli/test/utils_unit.test.ts b/cli/test/utils_unit.test.ts index 7bc3985a9c..47edf45236 100644 --- a/cli/test/utils_unit.test.ts +++ b/cli/test/utils_unit.test.ts @@ -4,7 +4,7 @@ */ import { expect, test, describe } from "bun:test"; -import { deepEqual, isFileResource, isFilesetResource, toCamel, capitalize } from "../src/utils/utils.ts"; +import { deepEqual, isFileResource, isFilesetResource, toCamel, capitalize, validateRequiredArgs } from "../src/utils/utils.ts"; import { getTypeStrFromPath, removeType, @@ -203,12 +203,12 @@ describe("removeType", () => { expect(removeType("f/test/my_var.variable.json", "variable")).toBe("f/test/my_var"); }); - test("throws for wrong type suffix", () => { - expect(() => removeType("f/test/my_var.variable.yaml", "resource")).toThrow(); + test("passes through path with wrong type suffix as clean path", () => { + expect(removeType("f/test/my_var.variable.yaml", "resource")).toBe("f/test/my_var.variable.yaml"); }); - test("throws for no type suffix", () => { - expect(() => removeType("f/test/my_script.ts", "variable")).toThrow(); + test("passes through path with no type suffix as clean path", () => { + expect(removeType("f/test/my_script.ts", "variable")).toBe("f/test/my_script.ts"); }); }); @@ -596,3 +596,103 @@ describe("removeExtensionToPath", () => { expect(removeExtensionToPath("f/test/api.fetch.ts")).toBe("f/test/api"); }); }); + +// ============================================================================= +// validateRequiredArgs +// ============================================================================= + +describe("validateRequiredArgs", () => { + test("throws when required args are missing", () => { + expect(() => + validateRequiredArgs({ required: ["name", "count"] }) + ).toThrow("Missing required arguments: name, count"); + }); + + test("does not throw when no required args", () => { + expect(() => validateRequiredArgs({ required: [] })).not.toThrow(); + }); + + test("does not throw for undefined schema", () => { + expect(() => validateRequiredArgs(undefined)).not.toThrow(); + expect(() => validateRequiredArgs(null)).not.toThrow(); + }); + + test("does not throw for schema without required field", () => { + expect(() => validateRequiredArgs({ type: "object", properties: {} })).not.toThrow(); + }); + + test("error message includes usage hint", () => { + try { + validateRequiredArgs({ required: ["name"] }); + } catch (e: any) { + expect(e.message).toContain('-d \'{"name":'); + } + }); +}); + +// ============================================================================= +// TarAsZip adapter +// ============================================================================= + +describe("TarAsZip adapter", () => { + // Import the adapter — it's not exported but we can test via tar creation + parsing + const { extract } = require("tar-stream"); + const { Readable } = require("node:stream"); + + // Helper: build a TarAsZip from entries via the actual class + async function buildTarAsZip(entries: Map) { + // Dynamically import to get the class + const pullModule = await import("../src/commands/sync/pull.ts"); + // TarAsZip is not exported, so we test indirectly via parseTarResponse + // Instead, test the tar creation → extraction round-trip + const { createTarBlob } = await import("../src/utils/tar.ts"); + + const tarEntries = Array.from(entries).map(([name, { content }]) => ({ + name, + content, + })); + const blob = await createTarBlob(tarEntries); + + // Parse via the same extract pattern used by TarAsZip + const buffer = Buffer.from(await blob.arrayBuffer()); + const result = new Map(); + const ex = extract(); + + return new Promise>((resolve, reject) => { + ex.on("entry", (header: any, stream: any, next: () => void) => { + const chunks: Buffer[] = []; + stream.on("data", (chunk: Buffer) => chunks.push(chunk)); + stream.on("end", () => { + result.set(header.name, { + content: Buffer.concat(chunks).toString("utf-8"), + isDir: header.type === "directory", + }); + next(); + }); + stream.on("error", reject); + stream.resume(); + }); + ex.on("finish", () => { + // Convert to simple map for assertions + const simpleMap = new Map(); + for (const [name, { content }] of result) { + simpleMap.set(name, content); + } + resolve(simpleMap); + }); + ex.on("error", reject); + Readable.from(buffer).pipe(ex); + }); + } + + test("tar round-trip preserves content", async () => { + const entries = new Map([ + ["f/scripts/hello.ts", { content: 'export async function main() { return "hello"; }', isDir: false }], + ["f/scripts/hello.script.yaml", { content: "summary: Hello\nkind: script\n", isDir: false }], + ]); + + const result = await buildTarAsZip(entries); + expect(result.get("f/scripts/hello.ts")).toBe('export async function main() { return "hello"; }'); + expect(result.get("f/scripts/hello.script.yaml")).toContain("summary: Hello"); + }); +}); diff --git a/cli/test/wmill_lock.test.ts b/cli/test/wmill_lock_unit.test.ts similarity index 100% rename from cli/test/wmill_lock.test.ts rename to cli/test/wmill_lock_unit.test.ts diff --git a/cli/test/workspace_conflicts.test.ts b/cli/test/workspace_conflicts_unit.test.ts similarity index 100% rename from cli/test/workspace_conflicts.test.ts rename to cli/test/workspace_conflicts_unit.test.ts diff --git a/cli/test/yaml_inline_tag.test.ts b/cli/test/yaml_inline_tag.test.ts new file mode 100644 index 0000000000..79638c7b50 --- /dev/null +++ b/cli/test/yaml_inline_tag.test.ts @@ -0,0 +1,58 @@ +/** + * Unit tests for custom !inline and !inline_fileset YAML tag handling. + * These tests require no backend — they test YAML parsing logic. + */ + +import { expect, test, describe } from "bun:test"; +import { yamlParseContent } from "../src/utils/yaml.ts"; +import { stringify as yamlStringify } from "yaml"; + +describe("YAML !inline tag resolution", () => { + test("unquoted !inline resolves to string with prefix", () => { + const result = yamlParseContent("test.yaml", "content: !inline get_users.ts"); + expect(result.content).toBe("!inline get_users.ts"); + }); + + test("quoted !inline is preserved as-is", () => { + const result = yamlParseContent("test.yaml", 'content: "!inline get_users.ts"'); + expect(result.content).toBe("!inline get_users.ts"); + }); + + test("unquoted and quoted produce identical results", () => { + const unquoted = yamlParseContent("test.yaml", "content: !inline script.ts"); + const quoted = yamlParseContent("test.yaml", 'content: "!inline script.ts"'); + expect(unquoted.content).toBe(quoted.content); + }); + + test("unquoted !inline_fileset resolves to string with prefix", () => { + const result = yamlParseContent("test.yaml", "value: !inline_fileset my_resource.fileset"); + expect(result.value).toBe("!inline_fileset my_resource.fileset"); + }); + + test("works within nested flow.yaml structure", () => { + const yaml = ` +value: + modules: + - id: a + value: + type: rawscript + content: !inline get_users.ts + language: bun + - id: b + value: + type: rawscript + content: !inline send_mail.ts + language: bun`; + const result = yamlParseContent("flow.yaml", yaml); + expect(result.value.modules[0].value.content).toBe("!inline get_users.ts"); + expect(result.value.modules[1].value.content).toBe("!inline send_mail.ts"); + }); + + test("round-trip: parse unquoted → stringify → parse preserves value", () => { + const yaml = "content: !inline my_script.ts"; + const parsed = yamlParseContent("test.yaml", yaml); + const serialized = yamlStringify(parsed); + const reparsed = yamlParseContent("test.yaml", serialized); + expect(reparsed.content).toBe("!inline my_script.ts"); + }); +}); diff --git a/cli/windmill-utils-internal/package-lock.json b/cli/windmill-utils-internal/package-lock.json index 57d295218f..e9ce5e3c9a 100644 --- a/cli/windmill-utils-internal/package-lock.json +++ b/cli/windmill-utils-internal/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-utils-internal", - "version": "1.3.5", + "version": "1.3.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-utils-internal", - "version": "1.3.5", + "version": "1.3.6", "license": "Apache 2.0", "devDependencies": { "@types/node": "^24.2.0", diff --git a/cli/windmill-utils-internal/package.json b/cli/windmill-utils-internal/package.json index d5c428b35f..07cf8b90b0 100644 --- a/cli/windmill-utils-internal/package.json +++ b/cli/windmill-utils-internal/package.json @@ -1,6 +1,6 @@ { "name": "windmill-utils-internal", - "version": "1.3.5", + "version": "1.3.7", "description": "Internal utility functions for Windmill", "main": "dist/cjs/index.js", "module": "dist/esm/index.js", diff --git a/cli/windmill-utils-internal/src/config/index.ts b/cli/windmill-utils-internal/src/config/index.ts index f3ae42b3c8..e23ba6ca86 100644 --- a/cli/windmill-utils-internal/src/config/index.ts +++ b/cli/windmill-utils-internal/src/config/index.ts @@ -1 +1 @@ -export * from "./config.ts"; \ No newline at end of file +export * from "./config"; \ No newline at end of file diff --git a/cli/windmill-utils-internal/src/index.ts b/cli/windmill-utils-internal/src/index.ts index 635893e2d1..da314a7c5a 100644 --- a/cli/windmill-utils-internal/src/index.ts +++ b/cli/windmill-utils-internal/src/index.ts @@ -8,8 +8,8 @@ * - Cross-platform path constants */ -export * from "./inline-scripts.ts"; -export * from "./path-utils.ts"; -export * from "./parse.ts"; -export * from "./config.ts"; -export { SEP, DELIMITER } from "./constants.ts"; \ No newline at end of file +export * from "./inline-scripts"; +export * from "./path-utils"; +export * from "./parse"; +export * from "./config"; +export { SEP, DELIMITER } from "./constants"; \ No newline at end of file diff --git a/cli/windmill-utils-internal/src/inline-scripts/extractor.ts b/cli/windmill-utils-internal/src/inline-scripts/extractor.ts index a3572ce7eb..e472372a99 100644 --- a/cli/windmill-utils-internal/src/inline-scripts/extractor.ts +++ b/cli/windmill-utils-internal/src/inline-scripts/extractor.ts @@ -1,5 +1,5 @@ -import { newPathAssigner, PathAssigner } from "../path-utils/path-assigner.ts"; -import { FlowModule, RawScript, ScriptLang } from "../gen/types.gen.ts"; +import { newPathAssigner, PathAssigner } from "../path-utils/path-assigner"; +import { FlowModule, RawScript, ScriptLang } from "../gen/types.gen"; /** * Represents an inline script extracted from a flow module @@ -23,14 +23,21 @@ function extractRawscriptInline( assigner: PathAssigner ): InlineScript[] { const [basePath, ext] = assigner.assignPath(summary ?? id, rawscript.language); - const path = mapping[id] ?? basePath + ext; + const mappedPath = mapping[id]; + const path = mappedPath ?? basePath + ext; const language = rawscript.language; const content = rawscript.content; const r = [{ path: path, content: content, language, is_lock: false}]; rawscript.content = "!inline " + path.replaceAll(separator, "/"); const lock = rawscript.lock; if (lock && lock != "") { - const lockPath = basePath + "lock"; + // Derive lock path base from the mapped content path when available, + // so lock files are named consistently with their content files. + const dotIdx = mappedPath ? mappedPath.lastIndexOf('.') : -1; + const lockBasePath = mappedPath + ? (dotIdx > 0 ? mappedPath.substring(0, dotIdx + 1) : mappedPath + '.') + : basePath; + const lockPath = lockBasePath + "lock"; rawscript.lock = "!inline " + lockPath.replaceAll(separator, "/"); r.push({ path: lockPath, content: lock, language, is_lock: true}); } @@ -191,7 +198,7 @@ export function extractCurrentMapping( } else if (m.value.type === "aiagent") { (m.value.tools ?? []).forEach((tool) => { const toolValue = tool.value; - if (!toolValue || toolValue.tool_type !== 'flowmodule' || toolValue.type !== 'rawscript' || !toolValue.content || !toolValue.content.startsWith("!inline")) { + if (!toolValue || toolValue.tool_type !== 'flowmodule' || toolValue.type !== 'rawscript' || !toolValue.content || !toolValue.content.startsWith("!inline ")) { return; } mapping[tool.id] = toolValue.content.trim().split(" ")[1]; diff --git a/cli/windmill-utils-internal/src/inline-scripts/index.ts b/cli/windmill-utils-internal/src/inline-scripts/index.ts index bb3c917dbb..eace8d3e4f 100644 --- a/cli/windmill-utils-internal/src/inline-scripts/index.ts +++ b/cli/windmill-utils-internal/src/inline-scripts/index.ts @@ -1,2 +1,2 @@ -export * from "./replacer.ts"; -export * from "./extractor.ts"; \ No newline at end of file +export * from "./replacer"; +export * from "./extractor"; \ No newline at end of file diff --git a/cli/windmill-utils-internal/src/inline-scripts/replacer.ts b/cli/windmill-utils-internal/src/inline-scripts/replacer.ts index 11b2cfaa1b..d926467b86 100644 --- a/cli/windmill-utils-internal/src/inline-scripts/replacer.ts +++ b/cli/windmill-utils-internal/src/inline-scripts/replacer.ts @@ -1,4 +1,4 @@ -import { AiAgent, FlowModule, FlowValue, RawScript } from "../gen/types.gen.ts"; +import { AiAgent, FlowModule, FlowValue, RawScript } from "../gen/types.gen"; export type LocalScriptInfo = { content: string; @@ -13,7 +13,8 @@ async function replaceRawscriptInline( fileReader: (path: string) => Promise, logger: { info: (message: string) => void; error: (message: string) => void }, separator: string, - removeLocks?: string[] + removeLocks?: string[], + missingFiles?: string[] ): Promise { if (!rawscript.content || !rawscript.content.startsWith("!inline")) { return; @@ -31,6 +32,7 @@ async function replaceRawscriptInline( rawscript.content = await fileReader(newPath); } catch { logger.error(`Script file ${newPath} not found`); + if (missingFiles) missingFiles.push(path); } } @@ -76,14 +78,14 @@ export async function replaceInlineScripts( localPath: string, separator: string = "/", removeLocks?: string[], - // renamer?: (path: string, newPath: string) => void, - // deleter?: (path: string) => void - ): Promise { + missingFiles?: string[], + ): Promise { + const missing = missingFiles ?? []; await Promise.all(modules.map(async (module) => { if (!module.value) { throw new Error(`Module value is undefined for module ${module.id}`); } - + if (module.value.type === "rawscript") { await replaceRawscriptInline( module.id, @@ -91,19 +93,20 @@ export async function replaceInlineScripts( fileReader, logger, separator, - removeLocks + removeLocks, + missing ); } else if (module.value.type === "forloopflow" || module.value.type === "whileloopflow") { - await replaceInlineScripts(module.value.modules, fileReader, logger, localPath, separator, removeLocks); + await replaceInlineScripts(module.value.modules, fileReader, logger, localPath, separator, removeLocks, missing); } else if (module.value.type === "branchall") { await Promise.all(module.value.branches.map(async (branch) => { - await replaceInlineScripts(branch.modules, fileReader, logger, localPath, separator, removeLocks); + await replaceInlineScripts(branch.modules, fileReader, logger, localPath, separator, removeLocks, missing); })); } else if (module.value.type === "branchone") { await Promise.all(module.value.branches.map(async (branch) => { - await replaceInlineScripts(branch.modules, fileReader, logger, localPath, separator, removeLocks); + await replaceInlineScripts(branch.modules, fileReader, logger, localPath, separator, removeLocks, missing); })); - await replaceInlineScripts(module.value.default, fileReader, logger, localPath, separator, removeLocks); + await replaceInlineScripts(module.value.default, fileReader, logger, localPath, separator, removeLocks, missing); } else if (module.value.type === "aiagent") { await Promise.all((module.value.tools ?? []).map(async (tool) => { const toolValue = tool.value; @@ -120,11 +123,13 @@ export async function replaceInlineScripts( fileReader, logger, separator, - removeLocks + removeLocks, + missing ); })); } })); + return missing; } /** diff --git a/cli/windmill-utils-internal/src/parse/index.ts b/cli/windmill-utils-internal/src/parse/index.ts index 41d09ed00d..fc26ce611a 100644 --- a/cli/windmill-utils-internal/src/parse/index.ts +++ b/cli/windmill-utils-internal/src/parse/index.ts @@ -1 +1 @@ -export * from "./parse-schema.ts"; \ No newline at end of file +export * from "./parse-schema"; \ No newline at end of file diff --git a/cli/windmill-utils-internal/src/path-utils/index.ts b/cli/windmill-utils-internal/src/path-utils/index.ts index 6f5c8d68be..ef23185664 100644 --- a/cli/windmill-utils-internal/src/path-utils/index.ts +++ b/cli/windmill-utils-internal/src/path-utils/index.ts @@ -1 +1 @@ -export * from "./path-assigner.ts"; \ No newline at end of file +export * from "./path-assigner"; \ No newline at end of file diff --git a/cli/windmill-utils-internal/src/path-utils/path-assigner.ts b/cli/windmill-utils-internal/src/path-utils/path-assigner.ts index 3fedbd8d37..07f423c876 100644 --- a/cli/windmill-utils-internal/src/path-utils/path-assigner.ts +++ b/cli/windmill-utils-internal/src/path-utils/path-assigner.ts @@ -1,4 +1,4 @@ -import { RawScript } from "../gen/types.gen.ts"; +import { RawScript } from "../gen/types.gen"; const INLINE_SCRIPT_PREFIX = "inline_script"; @@ -35,6 +35,7 @@ export const LANGUAGE_EXTENSIONS: Record = { duckdb: "duckdb.sql", bunnative: "ts", ruby: "rb", + rlang: "r", // for related places search: ADD_NEW_LANG }; @@ -111,6 +112,28 @@ export function getLanguageFromExtension( return undefined; } +/** + * Sanitizes a summary string for use as a filesystem-safe name. + * Removes or replaces characters that are invalid on common filesystems. + */ +const WINDOWS_RESERVED = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])$/; + +export function sanitizeForFilesystem(summary: string): string { + const name = summary + .toLowerCase() + .replaceAll(" ", "_") + // Remove characters invalid on Windows/Unix/Mac: / \ : * ? " < > | + // Also remove control characters (0x00-0x1F) and DEL (0x7F) + // deno-lint-ignore no-control-regex + .replace(/[/\\:*?"<>|\x00-\x1f\x7f]/g, "") + // Collapse consecutive underscores + .replace(/_+/g, "_") + // Trim leading/trailing dots and underscores (hidden files, Windows edge cases) + .replace(/^[._]+|[._]+$/g, ""); + // Prefix Windows reserved device names (CON, PRN, AUX, NUL, COM0-9, LPT0-9) + return WINDOWS_RESERVED.test(name) ? `_${name}` : name; +} + export interface PathAssigner { assignPath(summary: string | undefined, language: SupportedLanguage): [string, string]; } @@ -144,7 +167,7 @@ export function newPathAssigner(defaultTs: "bun" | "deno" | PathAssignerOptions, ): [string, string] { let name; - name = summary?.toLowerCase()?.replaceAll(" ", "_") ?? ""; + name = summary ? sanitizeForFilesystem(summary) : ""; let original_name = name; @@ -185,7 +208,7 @@ export function newRawAppPathAssigner(defaultTs: "bun" | "deno"): PathAssigner { ): [string, string] { let name; - name = summary?.toLowerCase()?.replaceAll(" ", "_") ?? ""; + name = summary ? sanitizeForFilesystem(summary) : ""; let original_name = name; diff --git a/cli/wmill.schema.json b/cli/wmill.schema.json new file mode 100644 index 0000000000..7129563f3f --- /dev/null +++ b/cli/wmill.schema.json @@ -0,0 +1,439 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "wmill.yaml", + "description": "Windmill CLI configuration file. Full reference: wmill config", + "type": "object", + "properties": { + "defaultTs": { + "type": "string", + "enum": [ + "bun", + "deno" + ], + "description": "Default TypeScript runtime for new scripts" + }, + "includes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Glob patterns for files to include in sync" + }, + "extraIncludes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Additional glob patterns merged with includes (useful in branch overrides)" + }, + "excludes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Glob patterns for files to exclude from sync" + }, + "skipVariables": { + "type": "boolean", + "description": "Skip syncing variables" + }, + "skipResources": { + "type": "boolean", + "description": "Skip syncing resources" + }, + "skipResourceTypes": { + "type": "boolean", + "description": "Skip syncing resource types" + }, + "skipSecrets": { + "type": "boolean", + "description": "Skip syncing secrets (true by default for security)" + }, + "skipScripts": { + "type": "boolean", + "description": "Skip syncing scripts" + }, + "skipFlows": { + "type": "boolean", + "description": "Skip syncing flows" + }, + "skipApps": { + "type": "boolean", + "description": "Skip syncing apps" + }, + "skipFolders": { + "type": "boolean", + "description": "Skip syncing folders" + }, + "skipWorkspaceDependencies": { + "type": "boolean", + "description": "Skip syncing workspace dependencies" + }, + "includeSchedules": { + "type": "boolean", + "description": "Include schedules in sync" + }, + "includeTriggers": { + "type": "boolean", + "description": "Include triggers (http, websocket, kafka, etc.) in sync" + }, + "includeUsers": { + "type": "boolean", + "description": "Include workspace users in sync" + }, + "includeGroups": { + "type": "boolean", + "description": "Include workspace groups in sync" + }, + "includeSettings": { + "type": "boolean", + "description": "Include workspace settings in sync" + }, + "includeKey": { + "type": "boolean", + "description": "Include encryption key in sync" + }, + "parallel": { + "type": "integer", + "description": "Number of parallel operations during sync" + }, + "locksRequired": { + "type": "boolean", + "description": "Require lock files for all scripts" + }, + "lint": { + "type": "boolean", + "description": "Run linting before push" + }, + "plainSecrets": { + "type": "boolean", + "description": "Handle secrets as plain text (not recommended)" + }, + "message": { + "type": "string", + "description": "Default commit message for sync operations" + }, + "promotion": { + "type": "string", + "description": "Branch name to use promotion overrides from during sync" + }, + "skipBranchValidation": { + "type": "boolean", + "description": "Skip validation that current git branch matches a configured branch" + }, + "nonDottedPaths": { + "type": "boolean", + "description": "Use __flow/__app/__raw_app suffixes instead of .flow/.app/.raw_app" + }, + "codebases": { + "type": "array", + "description": "Codebase bundling configurations for shared libraries", + "items": { + "type": "object", + "properties": { + "relative_path": { + "type": "string", + "description": "Path to the codebase directory" + }, + "includes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Glob patterns for files to include in bundle" + }, + "excludes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Glob patterns for files to exclude from bundle" + }, + "format": { + "type": "string", + "enum": [ + "cjs", + "esm" + ], + "description": "Bundle output format" + }, + "external": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Dependencies to leave unbundled (externals)" + }, + "assets": { + "type": "array", + "items": { + "type": "object", + "properties": { + "from": { + "type": "string" + }, + "to": { + "type": "string" + } + }, + "required": [ + "from", + "to" + ] + }, + "description": "Static files to copy into the bundle" + }, + "customBundler": { + "type": "string", + "description": "Path to a custom bundler script (replaces esbuild)" + }, + "inject": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Files to inject into every entry point" + }, + "define": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Compile-time constant definitions" + }, + "banner": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Text to prepend to output files by type" + }, + "loader": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "esbuild loader overrides by extension" + } + }, + "required": [ + "relative_path" + ], + "additionalProperties": false + } + }, + "gitBranches": { + "type": "object", + "description": "Map git branches to workspaces and per-branch sync overrides", + "properties": { + "commonSpecificItems": { + "type": "object", + "description": "Sync only specific items", + "properties": { + "variables": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific variable paths to sync" + }, + "resources": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific resource paths to sync" + }, + "triggers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific trigger paths to sync" + }, + "folders": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific folder paths to sync" + }, + "settings": { + "type": "boolean", + "description": "Whether to sync settings" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": { + "type": "object", + "properties": { + "baseUrl": { + "type": "string", + "description": "Windmill instance URL for this branch" + }, + "workspaceId": { + "type": "string", + "description": "Workspace ID to sync with for this branch" + }, + "overrides": { + "type": "object", + "description": "Override any top-level sync option for this branch" + }, + "promotionOverrides": { + "type": "object", + "description": "Overrides applied when using --promotion flag" + }, + "specificItems": { + "type": "object", + "description": "Sync only specific items", + "properties": { + "variables": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific variable paths to sync" + }, + "resources": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific resource paths to sync" + }, + "triggers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific trigger paths to sync" + }, + "folders": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific folder paths to sync" + }, + "settings": { + "type": "boolean", + "description": "Whether to sync settings" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "environments": { + "type": "object", + "description": "Alias for gitBranches — use if you prefer environment-based terminology", + "properties": { + "commonSpecificItems": { + "type": "object", + "description": "Sync only specific items", + "properties": { + "variables": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific variable paths to sync" + }, + "resources": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific resource paths to sync" + }, + "triggers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific trigger paths to sync" + }, + "folders": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific folder paths to sync" + }, + "settings": { + "type": "boolean", + "description": "Whether to sync settings" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": { + "type": "object", + "properties": { + "baseUrl": { + "type": "string", + "description": "Windmill instance URL for this branch" + }, + "workspaceId": { + "type": "string", + "description": "Workspace ID to sync with for this branch" + }, + "overrides": { + "type": "object", + "description": "Override any top-level sync option for this branch" + }, + "promotionOverrides": { + "type": "object", + "description": "Overrides applied when using --promotion flag" + }, + "specificItems": { + "type": "object", + "description": "Sync only specific items", + "properties": { + "variables": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific variable paths to sync" + }, + "resources": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific resource paths to sync" + }, + "triggers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific trigger paths to sync" + }, + "folders": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Specific folder paths to sync" + }, + "settings": { + "type": "boolean", + "description": "Whether to sync settings" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} diff --git a/docker-compose.yml b/docker-compose.yml index ffec3d5dda..8b636c7702 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -75,7 +75,6 @@ services: replicas: 3 resources: limits: - cpus: "1" memory: 2048M # for GB, use syntax '2Gi' restart: unless-stopped @@ -119,7 +118,6 @@ services: replicas: 1 resources: limits: - cpus: "1" memory: 2048M # for GB, use syntax '2Gi' restart: unless-stopped @@ -131,7 +129,6 @@ services: - MODE=worker - WORKER_GROUP=native - NATIVE_MODE=true - - NUM_WORKERS=8 - SLEEP_QUEUE=200 depends_on: db: @@ -147,7 +144,6 @@ services: # replicas: 1 # resources: # limits: - # cpus: "1" # memory: 2048M # # for GB, use syntax '2Gi' # restart: unless-stopped diff --git a/docker/DockerfileFull b/docker/DockerfileFull index 153f4fac5a..91ed372819 100644 --- a/docker/DockerfileFull +++ b/docker/DockerfileFull @@ -27,6 +27,10 @@ RUN /usr/bin/java -jar /usr/bin/coursier about # Ruby RUN apt-get install -y ruby ruby-bundler +# R +RUN apt-get install -y r-base-dev \ + && Rscript -e 'install.packages("renv", lib="/usr/lib/R/library", repos="https://cloud.r-project.org")' + # Fix UV cache permissions for non-root user support (uid 1000, etc.) # The uv tool install ansible command populates the UV cache with root-owned files RUN chmod -R a+rw /tmp/windmill/cache/uv && \ diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index f3bb9c0569..3fbb06da71 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -51,6 +51,10 @@ RUN /usr/bin/java -jar /usr/bin/coursier about # Ruby RUN apt-get install -y ruby ruby-bundler +# R +RUN apt-get install -y r-base-dev \ + && Rscript -e 'install.packages("renv", lib="/usr/lib/R/library", repos="https://cloud.r-project.org")' + # iptables RUN apt-get install -y iptables diff --git a/flake.nix b/flake.nix index 5f644d93dc..2bab1c120c 100644 --- a/flake.nix +++ b/flake.nix @@ -27,6 +27,16 @@ extensions = [ "rust-src" "rust-analyzer" "rustfmt" ]; }; + patchedClang = pkgs.llvmPackages_18.clang.overrideAttrs (oldAttrs: { + postFixup = '' + # Copy the original postFixup logic but skip add-hardening.sh + ${oldAttrs.postFixup or ""} + + # Remove the line that substitutes add-hardening.sh + sed -i 's/.*source.*add-hardening\.sh.*//' $out/bin/clang + ''; + }); + # --------------------------------------------------------------- # Native C/C++ dependencies (required to compile the backend) # --------------------------------------------------------------- @@ -72,14 +82,16 @@ version = "130.0.7"; target = stdenv.hostPlatform.rust.rustcTarget; sha256 = { - x86_64-linux = "sha256-pkdsuU6bAkcIHEZUJOt5PXdzK424CEgTLXjLtQ80t10="; + x86_64-linux = + "sha256-pkdsuU6bAkcIHEZUJOt5PXdzK424CEgTLXjLtQ80t10="; aarch64-linux = lib.fakeHash; x86_64-darwin = lib.fakeHash; aarch64-darwin = lib.fakeHash; }.${system}; in pkgs.fetchurl { name = "librusty_v8-${version}"; - url = "https://github.com/denoland/rusty_v8/releases/download/v${version}/librusty_v8_release_${target}.a.gz"; + url = + "https://github.com/denoland/rusty_v8/releases/download/v${version}/librusty_v8_release_${target}.a.gz"; inherit sha256; }; @@ -87,15 +99,28 @@ # pkg-config search path for native libraries # --------------------------------------------------------------- - pkgConfigPath = lib.makeSearchPath "lib/pkgconfig" - (with pkgs; [ openssl.dev libxml2.dev xmlsec.dev libxslt.dev cyrus_sasl.dev krb5.dev ]); + pkgConfigPath = lib.makeSearchPath "lib/pkgconfig" (with pkgs; [ + openssl.dev + libxml2.dev + xmlsec.dev + libxslt.dev + cyrus_sasl.dev + krb5.dev + ]); # --------------------------------------------------------------- # RPATH — embed Nix store library paths into compiled binaries # --------------------------------------------------------------- rpathLibs = lib.makeLibraryPath (with pkgs; [ - openssl libffi cyrus_sasl krb5 libxml2 xmlsec libxslt stdenv.cc.cc.lib + openssl + libffi + cyrus_sasl + krb5 + libxml2 + xmlsec + libxslt + stdenv.cc.cc.lib ]); # --------------------------------------------------------------- @@ -113,11 +138,17 @@ (builtins.readFile "${stdenv.cc}/nix-support/libcxx-cxxflags") "-idirafter ${pkgs.libiconv}/include" ] ++ lib.optionals stdenv.cc.isClang [ - "-idirafter ${stdenv.cc.cc}/lib/clang/${lib.getVersion stdenv.cc.cc}/include" + "-idirafter ${stdenv.cc.cc}/lib/clang/${ + lib.getVersion stdenv.cc.cc + }/include" ] ++ lib.optionals stdenv.cc.isGNU [ "-isystem ${stdenv.cc.cc}/include/c++/${lib.getVersion stdenv.cc.cc}" - "-isystem ${stdenv.cc.cc}/include/c++/${lib.getVersion stdenv.cc.cc}/${stdenv.hostPlatform.config}" - "-idirafter ${stdenv.cc.cc}/lib/gcc/${stdenv.hostPlatform.config}/${lib.getVersion stdenv.cc.cc}/include" + "-isystem ${stdenv.cc.cc}/include/c++/${ + lib.getVersion stdenv.cc.cc + }/${stdenv.hostPlatform.config}" + "-idirafter ${stdenv.cc.cc}/lib/gcc/${stdenv.hostPlatform.config}/${ + lib.getVersion stdenv.cc.cc + }/include" ]); # --------------------------------------------------------------- @@ -131,12 +162,16 @@ BINDGEN_EXTRA_CLANG_ARGS = bindgenClangArgs; # Force clang 18 as cargo linker (stdenv may bring a newer clang that causes SIGSEGV with mold) - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER = "${pkgs.llvmPackages_18.clang}/bin/clang"; - CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER = "${pkgs.llvmPackages_18.clang}/bin/clang"; + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER = + "${pkgs.llvmPackages_18.clang}/bin/clang"; + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER = + "${pkgs.llvmPackages_18.clang}/bin/clang"; # Embed rpath so binaries find Nix store .so files at runtime - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "-C link-arg=-fuse-ld=mold -C link-arg=-Wl,-rpath,${rpathLibs}"; - CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "-C link-arg=-fuse-ld=mold -C link-arg=-Wl,-rpath,${rpathLibs}"; + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = + "-C link-arg=-fuse-ld=mold -C link-arg=-Wl,-rpath,${rpathLibs}"; + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS = + "-C link-arg=-fuse-ld=mold -C link-arg=-Wl,-rpath,${rpathLibs}"; CARGO_HOST_RUSTFLAGS = "-C link-arg=-Wl,-rpath,${rpathLibs}"; # https://github.com/NixOS/nixpkgs/issues/370494 — jemalloc build fix @@ -197,6 +232,10 @@ hash = "sha256-8E0WtDFc7RcqmftDigMyy1xXUkjgL4X4kpf7h1GdE48="; }; + rWithPackages = pkgs.rWrapper.override { + packages = with pkgs.rPackages; [ renv ]; + }; + extraRuntimes = with pkgs; [ dotnet-sdk_9 php @@ -222,6 +261,7 @@ ANSIBLE_PLAYBOOK_PATH = "${pkgs.ansible}/bin/ansible-playbook"; ANSIBLE_GALAXY_PATH = "${pkgs.ansible}/bin/ansible-galaxy"; CARGO_SWEEP_PATH = "${pkgs.cargo-sweep}/bin/cargo-sweep"; + RSCRIPT_PATH = "${rWithPackages}/bin/Rscript"; }; # --------------------------------------------------------------- @@ -251,13 +291,23 @@ (pkgs.writeScriptBin "wm" '' cd ./frontend npm install - npm run ${if stdenv.isDarwin then "generate-backend-client-mac" else "generate-backend-client"} + npm run ${ + if stdenv.isDarwin then + "generate-backend-client-mac" + else + "generate-backend-client" + } npm run dev "$@" '') (pkgs.writeScriptBin "wm-build" '' cd ./frontend npm install - npm run ${if stdenv.isDarwin then "generate-backend-client-mac" else "generate-backend-client"} + npm run ${ + if stdenv.isDarwin then + "generate-backend-client-mac" + else + "generate-backend-client" + } npm run build "$@" '') (pkgs.writeScriptBin "wm-migrate" '' @@ -322,22 +372,20 @@ # Shared inputs and settings for default + full shells # --------------------------------------------------------------- - coreBuildInputs = nativeBuildDeps ++ commonRuntimes ++ [ - rustStable - openapi-generator-cli - ] ++ (with pkgs; [ - nodejs - git - sqlx-cli - cargo-watch - jq - gnused + coreBuildInputs = nativeBuildDeps ++ commonRuntimes + ++ [ rustStable openapi-generator-cli ] ++ (with pkgs; [ + nodejs + git + sqlx-cli + cargo-watch + jq + gnused - # CLI tools (for AI agents and dev workflow) - gh - asciinema - mermaid-cli - ]); + # CLI tools (for AI agents and dev workflow) + gh + asciinema + mermaid-cli + ]); # Playwright: use Nix-provided browsers (version-matched to playwright-driver) # Mermaid/Puppeteer: point at Nix chromium (Puppeteer respects this env var) @@ -380,16 +428,26 @@ sandboxEnv = pkgs.buildEnv { name = "windmill-sandbox"; - paths = coreBuildInputs ++ helperScriptsBase - ++ [ playwrightWrapper sandboxEnvScript pkgConfigWrapper pkgs.chromium ]; + paths = coreBuildInputs ++ helperScriptsBase ++ [ + playwrightWrapper + sandboxEnvScript + pkgConfigWrapper + pkgs.chromium + ]; }; sandboxFullEnv = pkgs.buildEnv { name = "windmill-sandbox-full"; - paths = coreBuildInputs ++ extraRuntimes - ++ helperScriptsBase ++ helperScriptsFull - ++ [ playwrightWrapper sandboxEnvScript pkgConfigWrapper pkgs.chromium - pkgs.cargo-sweep pkgs.xcaddy pkgs.nsjail ]; + paths = coreBuildInputs ++ extraRuntimes ++ helperScriptsBase + ++ helperScriptsFull ++ [ + playwrightWrapper + sandboxEnvScript + pkgConfigWrapper + pkgs.chromium + pkgs.cargo-sweep + pkgs.xcaddy + pkgs.nsjail + ]; }; in { @@ -412,8 +470,8 @@ shellHook = devShellHook; buildInputs = coreBuildInputs; - packages = helperScriptsBase ++ [ playwrightWrapper ]; - }); + packages = helperScriptsBase ++ [ playwrightWrapper ]; + }); # ============================================================= # full — all language runtimes, k8s tooling, specialized scripts @@ -428,27 +486,28 @@ pyright openapi-python-client - # LSP / editor - svelte-language-server - taplo + # LSP / editor + svelte-language-server + taplo - # Extra dev tools - cargo-sweep + # Extra dev tools + cargo-sweep - # Kubernetes - minikube - kubectl - kubernetes-helm - conntrack-tools - cri-tools + # Kubernetes + minikube + kubectl + kubernetes-helm + conntrack-tools + cri-tools - # Extra - xcaddy - nsjail - ]); + # Extra + xcaddy + nsjail + ]); - packages = helperScriptsBase ++ helperScriptsFull ++ [ playwrightWrapper ]; - }); + packages = helperScriptsBase ++ helperScriptsFull + ++ [ playwrightWrapper ]; + }); # ============================================================= # wasm — WASM target compilation (nightly Rust) @@ -458,15 +517,23 @@ devShells.wasm = pkgs.mkShell (buildEnvVars // { hardeningDisable = [ "all" ]; + # Explicitly set paths for headers and linker + # DO NOT REMOVE - if absent, breaks wasm builds on NixOS. + shellHook = '' + export CC=${patchedClang}/bin/clang + ''; + buildInputs = nativeBuildDeps ++ (with pkgs; [ (rust-bin.nightly.latest.default.override { extensions = [ "rust-src" "rust-analyzer" ]; - targets = [ "wasm32-unknown-unknown" "wasm32-unknown-emscripten" ]; + targets = + [ "wasm32-unknown-unknown" "wasm32-unknown-emscripten" ]; }) wasm-pack deno emscripten nushell + nodejs glibc_multi ]); }); diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 914eb790d1..649bba1bef 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.665.0", + "version": "1.672.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.665.0", + "version": "1.672.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -83,10 +83,12 @@ "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", "windmill-parser-wasm-py": "1.657.2", + "windmill-parser-wasm-r": "1.668.1", "windmill-parser-wasm-regex": "1.653.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.647.1", "windmill-parser-wasm-ts": "1.657.2", + "windmill-parser-wasm-wac": "1.668.6", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", "windmill-utils-internal": "^1.3.4", @@ -162,8 +164,8 @@ } }, "../backend/parsers/windmill-parser-wasm/pkg-ts": { - "name": "windmill-parser-wasm", - "version": "1.654.0" + "name": "windmill-parser-wasm-ts", + "version": "1.589.3" }, "node_modules/@alloc/quick-lru": { "version": "5.2.0", @@ -12151,9 +12153,9 @@ } }, "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", "optional": true, @@ -13684,6 +13686,11 @@ "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.657.2.tgz", "integrity": "sha512-3CN2rziafgCWcZri812+CkzuaE3P3/7dXmV9lSDpK9ma6Esd4zkHRXUFSyRzQE/R7Fxj5mSmSNX6xTff8eX5mw==" }, + "node_modules/windmill-parser-wasm-r": { + "version": "1.668.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-r/-/windmill-parser-wasm-r-1.668.1.tgz", + "integrity": "sha512-5YNeUibxpNBvYrxCgQcz1PxGhTFx2CyEpg2udtIhq7bx0d4gF/KDZVupMeQmAObmrEtTSFGUWNRJ4zXSWNrSpQ==" + }, "node_modules/windmill-parser-wasm-regex": { "version": "1.653.0", "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.653.0.tgz", @@ -13704,6 +13711,11 @@ "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.657.2.tgz", "integrity": "sha512-tiOUVsMKTc85m/a2BKpgAN3xTz+OPrUhcjEBPJNTdzrQOir1G5WeNkQ403BW+d1qI0BAVWT0gZnJ+4AhavBP+w==" }, + "node_modules/windmill-parser-wasm-wac": { + "version": "1.668.6", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-wac/-/windmill-parser-wasm-wac-1.668.6.tgz", + "integrity": "sha512-/ovcLWlIO+TQMrnWwcWoXquJI6hTZ+Zpo4POhV8z6e+pb4cVModGysASI9mERn2e9uj8/xSRmYqk54TLG8oUbQ==" + }, "node_modules/windmill-parser-wasm-yaml": { "version": "1.593.0", "resolved": "https://registry.npmjs.org/windmill-parser-wasm-yaml/-/windmill-parser-wasm-yaml-1.593.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index dadb95ca8e..dc9f9e0177 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.665.0", + "version": "1.672.0", "scripts": { "dev": "vite dev", "build": "vite build", @@ -156,10 +156,12 @@ "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", "windmill-parser-wasm-py": "1.657.2", + "windmill-parser-wasm-r": "1.668.1", "windmill-parser-wasm-regex": "1.653.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.647.1", "windmill-parser-wasm-ts": "1.657.2", + "windmill-parser-wasm-wac": "1.668.6", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", "windmill-utils-internal": "^1.3.4", @@ -295,6 +297,10 @@ "svelte": "./package/components/recording/ScriptRecordingReplay.svelte", "default": "./package/components/recording/ScriptRecordingReplay.svelte" }, + "./components/recording/types": { + "types": "./package/components/recording/types.d.ts", + "default": "./package/components/recording/types.js" + }, "./components/FlowWrapper.svelte": { "types": "./package/components/FlowWrapper.svelte.d.ts", "svelte": "./package/components/FlowWrapper.svelte", @@ -500,6 +506,9 @@ "components/ScriptRecordingReplay.svelte": [ "./package/components/recording/ScriptRecordingReplay.svelte.d.ts" ], + "components/recording/types": [ + "./package/components/recording/types.d.ts" + ], "components/FlowBuilder.svelte": [ "./package/components/FlowBuilder.svelte.d.ts" ], diff --git a/frontend/src/lib/components/AddUser.svelte b/frontend/src/lib/components/AddUser.svelte index 42f0e0430a..3a76bb96e1 100644 --- a/frontend/src/lib/components/AddUser.svelte +++ b/frontend/src/lib/components/AddUser.svelte @@ -1,6 +1,6 @@ @@ -80,15 +91,27 @@ {/snippet} {#snippet content()} -
+
Add a new user - Email - + {#if isServiceAccount} + Username + + {:else} + Email + - {#if !automateUsernameCreation} - Username - + {#if !automateUsernameCreation} + Username + + {/if} {/if} Role @@ -112,6 +135,13 @@ tooltip="An admin has full control over a specific Windmill workspace, including the ability to manage users, edit entities, and control permissions within the workspace." {item} /> + {/snippet} diff --git a/frontend/src/lib/components/ArgInfo.svelte b/frontend/src/lib/components/ArgInfo.svelte index c99f994728..a1cfdf6276 100644 --- a/frontend/src/lib/components/ArgInfo.svelte +++ b/frontend/src/lib/components/ArgInfo.svelte @@ -74,7 +74,15 @@ +{:else if isString(value) && value.startsWith('$jsonvar:')} + diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 197275fbc1..56d0d4b0fa 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -495,6 +495,8 @@ let { debounced, clearDebounce } = debounce(() => compareValues(value), 50) let inputCat = $derived(computeInputCat(type, format, itemsType?.type, enum_, contentEncoding)) + let isNonStringSecret = $derived((password || extra?.['password'] == true) && type === 'object') + let displayJsonToggleHeader = $derived( displayHeader && inputCat === 'list' && @@ -558,6 +560,12 @@ class="text-accent underline font-normal" onclick={() => variableEditor?.editVariable?.(value.slice(5))}>{value.slice(5)} + {:else if value && typeof value == 'string' && value?.startsWith('$jsonvar:')} + Linked to variable {/if}
{/if} @@ -1436,12 +1444,13 @@ {:else} {/if} {:else} - + {/if} {:else} {#key extra?.['minRows']} @@ -1487,6 +1496,18 @@ {@render actions?.()}
+ {#if isNonStringSecret} + {#if typeof value === 'string' && value.startsWith('$jsonvar:')} +
+ Sensitive — stored as secret: {value.slice('$jsonvar:'.length)} +
+ {:else} +
Sensitive — will be stored as secret on submit
+ {/if} + {/if} + {#if !compact || (error && error != '')}
{#if disabled || error === ''} diff --git a/frontend/src/lib/components/CompareWorkspaces.svelte b/frontend/src/lib/components/CompareWorkspaces.svelte index 6905ed6931..4f9184c93c 100644 --- a/frontend/src/lib/components/CompareWorkspaces.svelte +++ b/frontend/src/lib/components/CompareWorkspaces.svelte @@ -144,7 +144,7 @@ const app = await AppService.getAppByPath({ workspace, path }) return app.summary } else if (kind === 'folder') { - const folder = await FolderService.getFolder({ workspace, name: path.slice(2) }) + const folder = await FolderService.getFolder({ workspace, name: path.replace(/^f\//, '') }) return folder.summary } } catch (error) { @@ -361,7 +361,14 @@ const parent = parentWorkspaceId const current = currentWorkspaceId - for (const itemKey of selectedItems) { + const sortedItems = [...selectedItems].sort((a, b) => { + const aIsFolder = a.startsWith('folder:') + const bIsFolder = b.startsWith('folder:') + if (aIsFolder && !bIsFolder) return -1 + if (!aIsFolder && bIsFolder) return 1 + return 0 + }) + for (const itemKey of sortedItems) { const diff = selectableDiffs.find((d) => itemKey == getItemKey(d)) if (!diff) { diff --git a/frontend/src/lib/components/DefaultTagsInner.svelte b/frontend/src/lib/components/DefaultTagsInner.svelte index ab2d403ec8..e57afd9659 100644 --- a/frontend/src/lib/components/DefaultTagsInner.svelte +++ b/frontend/src/lib/components/DefaultTagsInner.svelte @@ -4,7 +4,11 @@ import { SettingService, WorkerService, WorkspaceService } from '$lib/gen' import { sendUserToast } from '$lib/toast' import { enterpriseLicense, superadmin } from '$lib/stores' - import { DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING } from '$lib/consts' + import { + DEFAULT_TAGS_PER_WORKSPACE_SETTING, + DEFAULT_TAGS_WORKSPACES_SETTING, + PREVIEW_TAGS_OVERRIDE_SETTING + } from '$lib/consts' import Toggle from './Toggle.svelte' import MultiSelect from './select/MultiSelect.svelte' import { safeSelectItems } from './select/utils.svelte' @@ -22,16 +26,19 @@ let defaultTags = $state(undefined) let limitToWorkspaces = $state(false) + let previewTagsOverride = $state(false) // Change detection let originalDefaultTagPerWorkspace = $state(defaultTagPerWorkspace) let originalDefaultTagWorkspaces = $state(defaultTagWorkspaces) + let originalPreviewTagsOverride = $state(false) // Detect changes let hasChanges = $derived( originalDefaultTagPerWorkspace !== defaultTagPerWorkspace || JSON.stringify($state.snapshot(originalDefaultTagWorkspaces)?.sort() || []) !== - JSON.stringify($state.snapshot(defaultTagWorkspaces)?.sort() || []) + JSON.stringify($state.snapshot(defaultTagWorkspaces)?.sort() || []) || + originalPreviewTagsOverride !== previewTagsOverride ) let workspaces: string[] = $state([]) @@ -47,6 +54,11 @@ key: DEFAULT_TAGS_WORKSPACES_SETTING })) as any) ?? [] limitToWorkspaces = defaultTagWorkspaces ? defaultTagWorkspaces.length > 0 : false + previewTagsOverride = + ((await SettingService.getGlobal({ + key: PREVIEW_TAGS_OVERRIDE_SETTING + })) as any) ?? false + originalPreviewTagsOverride = previewTagsOverride } catch (err) { sendUserToast(`Could not load default tags: ${err}`, true) } @@ -68,10 +80,17 @@ : undefined } }) + await SettingService.setGlobal({ + key: PREVIEW_TAGS_OVERRIDE_SETTING, + requestBody: { + value: previewTagsOverride + } + }) // Update original state after save originalDefaultTagPerWorkspace = defaultTagPerWorkspace originalDefaultTagWorkspaces = [...(defaultTagWorkspaces || [])] + originalPreviewTagsOverride = previewTagsOverride loadDefaultTags() sendUserToast('Saved') @@ -146,6 +165,18 @@ /> {/if} {/if} +
+ +
@@ -168,6 +199,17 @@
{/each} + {#if previewTagsOverride} +
+
+ preview +
+
→
+
+ {defaultTagPerWorkspace ? 'preview-$workspace' : 'preview'} +
+
+ {/if} {/if} diff --git a/frontend/src/lib/components/DeployWorkspace.svelte b/frontend/src/lib/components/DeployWorkspace.svelte index 75b5e6987c..d3ae70959f 100644 --- a/frontend/src/lib/components/DeployWorkspace.svelte +++ b/frontend/src/lib/components/DeployWorkspace.svelte @@ -266,13 +266,25 @@ return getTriggerDependency(additionalInformation.triggers.kind, path, $workspaceStore!) } throw new Error('Missing trigger information') + } else if (kind == 'script') { + const imports = await WorkspaceService.getImports({ + workspace: $workspaceStore!, + importerPath: path + }) + return imports.map((importedPath) => ({ kind: 'script' as Kind, path: importedPath })) } return [] } let toProcess = [{ kind, path }] + let processedSet = new Set() let processed: { kind: Kind; path: string }[] = [] while (toProcess.length > 0) { const { kind, path } = toProcess.pop()! + const key = `${kind}:${path}` + if (processedSet.has(key)) { + continue + } + processedSet.add(key) toProcess.push(...(await rec(kind, path))) processed.push({ kind, path }) } diff --git a/frontend/src/lib/components/EditableSchemaForm.svelte b/frontend/src/lib/components/EditableSchemaForm.svelte index 989ce6a2fe..85b8abf753 100644 --- a/frontend/src/lib/components/EditableSchemaForm.svelte +++ b/frontend/src/lib/components/EditableSchemaForm.svelte @@ -51,6 +51,7 @@ noPreview?: boolean jsonEnabled?: boolean isAppInput?: boolean + showSensitiveToggle?: boolean displayWebhookWarning?: boolean onlyMaskPassword?: boolean editTab: @@ -95,6 +96,7 @@ noPreview = false, jsonEnabled = true, isAppInput = false, + showSensitiveToggle = false, displayWebhookWarning = false, onlyMaskPassword = false, editTab, @@ -297,7 +299,9 @@ } const editTabDefaultSize = untrack(() => noPreview) ? 100 : 50 - editPanelSize = untrack(() => editTab) ? (untrack(() => editPanelInitialSize) ?? editTabDefaultSize) : 0 + editPanelSize = untrack(() => editTab) + ? (untrack(() => editPanelInitialSize) ?? editTabDefaultSize) + : 0 let inputPanelSize = $state(100 - editPanelSize) let editPanelSizeSmooth = tweened(editPanelSize, { duration: 150 @@ -677,6 +681,7 @@ bind:order={schema.properties[argName].order} {isFlowInput} {isAppInput} + {showSensitiveToggle} > {#snippet typeeditor()} {#if isFlowInput || isAppInput} diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index ee2108711c..89f8b803a2 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -79,7 +79,16 @@ iconOnly?: boolean validCode?: boolean kind?: 'script' | 'trigger' | 'approval' - template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' | 'claudesandbox' | 'wac_python' | 'wac_typescript' + template?: + | 'pgsql' + | 'mysql' + | 'script' + | 'docker' + | 'powershell' + | 'bunnative' + | 'claudesandbox' + | 'wac_python' + | 'wac_typescript' collabMode?: boolean collabLive?: boolean collabUsers?: { name: string }[] @@ -147,6 +156,7 @@ 'nu', 'java', 'ruby', + 'rlang', 'postgresql', 'mysql', 'bigquery', @@ -173,7 +183,8 @@ 'csharp', 'nu', 'java', - 'ruby' + 'ruby', + 'rlang' // for related places search: ADD_NEW_LANG ].includes(lang ?? '') ) @@ -193,7 +204,8 @@ 'csharp', 'nu', 'java', - 'ruby' + 'ruby', + 'rlang' // for related places search: ADD_NEW_LANG ].includes(lang ?? '') ) @@ -506,6 +518,8 @@ // for related places search: ADD_NEW_LANG } else if (lang == 'ruby') { editor.insertAtCursor(`ENV['${name}']`) + } else if (lang == 'rlang') { + editor.insertAtCursor(`Sys.getenv("${name}")`) } else if ( ['postgresql', 'mysql', 'bigquery', 'mssql', 'oracledb', 'snowflake', 'duckdb'].includes( lang ?? '' @@ -574,6 +588,8 @@ string ${windmillPathToCamelCaseName(path)} = await client.GetStringAsync(uri); editor.insertAtBeginning("require 'windmill/mini'\n") } editor.insertAtCursor(`get_variable("${path}")`) + } else if (lang == 'rlang') { + editor.insertAtCursor(`get_variable("${path}")`) } sendUserToast(`${name} inserted at cursor`) }} @@ -653,6 +669,8 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS editor.insertAtBeginning("require 'windmill/mini'\n") } editor.insertAtCursor(`get_resource("${path}")`) + } else if (lang == 'rlang') { + editor.insertAtCursor(`get_resource("${path}")`) } else if (lang == 'duckdb') { let t = { postgresql: 'postgres', mysql: 'mysql', bigquery: 'bigquery' }[resType] if (!t) { @@ -739,7 +757,7 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS startIcon={{ icon: Settings }} target="_blank" variant="accent" - href="{base}/workspace_settings?tab=windmill_lfs" + href="{base}/workspace_settings?tab=ducklake" > Go to settings diff --git a/frontend/src/lib/components/FlowGraphViewer.svelte b/frontend/src/lib/components/FlowGraphViewer.svelte index 9bfbca3ee4..7e4fceeda7 100644 --- a/frontend/src/lib/components/FlowGraphViewer.svelte +++ b/frontend/src/lib/components/FlowGraphViewer.svelte @@ -1,7 +1,10 @@ -
+
{#if !noGraph}
@@ -57,7 +80,7 @@ cache={flow.value.cache_ttl !== undefined} path={flow?.path} {download} - {minHeight} + minHeight={fillAvailableHeight ? Math.max(minHeight, availableHeight) : minHeight} {workspace} modules={flow?.value?.modules} failureModule={flow?.value?.failure_module} @@ -81,14 +104,16 @@ />
{/if} - {#if !noSide} + {#if !noSide && !(hideDefaultInputs && stepDetail == undefined)} {/if}
diff --git a/frontend/src/lib/components/FlowGraphViewerStep.svelte b/frontend/src/lib/components/FlowGraphViewerStep.svelte index 8c560abef0..5ca73d7d1f 100644 --- a/frontend/src/lib/components/FlowGraphViewerStep.svelte +++ b/frontend/src/lib/components/FlowGraphViewerStep.svelte @@ -23,9 +23,10 @@ schema?: any | undefined stepDetail?: FlowModule | string | undefined jobScriptHash?: string | undefined + hideDefaultInputs?: boolean } - let { schema = undefined, stepDetail = undefined, jobScriptHash = undefined }: Props = $props() + let { schema = undefined, stepDetail = undefined, jobScriptHash = undefined, hideDefaultInputs = false }: Props = $props() let codeViewer: Drawer | undefined = $state() @@ -50,17 +51,10 @@
- {#if stepDetail.value.path.startsWith('hub/')} -
-

Code

- -
- {/if} +
+

Code

+ +
{:else if stepDetail.value.type == 'rawscript'}

Step inputs

@@ -92,10 +86,10 @@
{#if stepDetail == undefined}
-

+

Click on a step to see its details

- {#if schema} + {#if schema && !hideDefaultInputs}

Flow Inputs

{/if} @@ -217,27 +211,16 @@
{/if} - {#if stepDetail.value.path.startsWith('hub/')} -
-
-

Code

- -
- -
- {:else} - - {/if} +
+

Code

+ +
+ {:else if stepDetail.value.type == 'aiagent'}

Step inputs

diff --git a/frontend/src/lib/components/FlowMetadata.svelte b/frontend/src/lib/components/FlowMetadata.svelte index 3d4be84545..022fa874a2 100644 --- a/frontend/src/lib/components/FlowMetadata.svelte +++ b/frontend/src/lib/components/FlowMetadata.svelte @@ -2,6 +2,7 @@ import { type Job } from '$lib/gen' import { base } from '$lib/base' import JobStatus from '$lib/components/JobStatus.svelte' + import { flowPathToHref } from '$lib/scripts' import { displayDate, truncateRev } from '$lib/utils' import ScheduleEditor from '$lib/components/triggers/schedules/ScheduleEditor.svelte' import TimeAgo from './TimeAgo.svelte' @@ -94,7 +95,9 @@ {#if (job && job.job_kind == 'flow') || job?.job_kind == 'script'} {@const stem = `${job?.job_kind}s`} {@const isScript = job?.job_kind === 'script'} - {@const viewHref = `${base}/${stem}/get/${isScript ? job?.script_hash : job?.script_path}`} + {@const viewHref = isScript + ? `${base}/${stem}/get/${job?.script_hash}` + : flowPathToHref(job?.script_path ?? '')}
{#if isScript} diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index 9c4ed4231e..01ae4c53fa 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -11,6 +11,7 @@ import { createEventDispatcher, getContext, untrack } from 'svelte' import type { FlowEditorContext } from './flows/types' import { runFlowPreview } from './flows/utils.svelte' + import { processSecretArgs } from './secretArgUtils' import SchemaForm from './SchemaForm.svelte' import SchemaFormWithArgPicker from './SchemaFormWithArgPicker.svelte' import FlowStatusViewer from '../components/FlowStatusViewer.svelte' @@ -171,6 +172,7 @@ lastPreviewFlow = JSON.stringify(flowStore.val) flowProgressBar?.reset() const newFlow = extractFlow(previewMode) + args = await processSecretArgs(args, flowStore.val.schema as any) newJobId = await runFlowPreview(args, newFlow, $pathStore, restartedFrom, conversationId) jobId = newJobId isRunning = true diff --git a/frontend/src/lib/components/FlowStatusViewer.svelte b/frontend/src/lib/components/FlowStatusViewer.svelte index 1642193a46..4b4e08c1b9 100644 --- a/frontend/src/lib/components/FlowStatusViewer.svelte +++ b/frontend/src/lib/components/FlowStatusViewer.svelte @@ -44,6 +44,7 @@ workspaceId = undefined, flowState = $bindable({}), selectedJobStep = $bindable(undefined), + hideFlowResult = false, hideTimeline = false, hideDownloadInGraph = false, hideNodeDefinition = false, @@ -175,6 +176,7 @@ } }} {showLogsWithResult} + {hideFlowResult} notes={notesProp} groups={groupsProp} /> diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 8aaeadb5ef..d35ae53c47 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -136,6 +136,7 @@ } showLogsWithResult?: boolean showJobDetailHeader?: boolean + hideFlowResult?: boolean notes?: FlowNote[] groups?: FlowValue['groups'] } @@ -178,6 +179,7 @@ toolCallStore, showLogsWithResult = false, showJobDetailHeader = false, + hideFlowResult = false, notes: notesProp = undefined, groups: groupsProp = undefined }: Props = $props() @@ -1356,7 +1358,7 @@ />
{/if} - {:else if render} + {:else if render && !hideFlowResult}
{#if showLogsWithResult && job} @@ -2141,7 +2143,7 @@ likely did not run yet

{/if} - {:else}

Select a node to see its details here

{/if}
@@ -2157,7 +2159,7 @@ {#if node?.job_id} {:else} -
Select a node with a job to see HTTP request traces
{/if} diff --git a/frontend/src/lib/components/FlowStatusWaitingForEvents.svelte b/frontend/src/lib/components/FlowStatusWaitingForEvents.svelte index 195a1d6fad..1b219153da 100644 --- a/frontend/src/lib/components/FlowStatusWaitingForEvents.svelte +++ b/frontend/src/lib/components/FlowStatusWaitingForEvents.svelte @@ -3,7 +3,7 @@ import { type Job, JobService } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' - import { X } from 'lucide-svelte' + import { ExternalLink, X } from 'lucide-svelte' import DisplayResult from './DisplayResult.svelte' import Tooltip from './Tooltip.svelte' import { Button } from './common' @@ -23,6 +23,7 @@ let default_payload: object = $state({}) let description: any = $state(undefined) let hide_cancel = $state(false) + let approvalPageUrl: string | undefined = $state(undefined) let defaultValues = $state({}) @@ -47,6 +48,8 @@ defaultValues = JSON.parse(JSON.stringify(args)) default_payload = args + approvalPageUrl = job_result?.['approvalPage'] + actionTaken = false hide_cancel = job?.raw_flow?.modules?.[approvalStep]?.suspend?.hide_cancel ?? false schema = mergeSchema( job?.raw_flow?.modules?.[approvalStep]?.suspend?.resume_form?.schema ?? {}, @@ -55,6 +58,7 @@ } let loading = $state(false) + let actionTaken = $state(false) async function continu(approve: boolean) { loading = true try { @@ -66,6 +70,7 @@ approved: approve } }) + actionTaken = true } catch (e: any) { sendUserToast(e?.body ?? e?.message ?? 'Failed', true) } finally { @@ -84,7 +89,7 @@
{/if}
-
+
{#if !hide_cancel}
{/if}
-
+ {#if approvalPageUrl} + + Approval page + + {/if} + {#if job?.raw_flow?.modules?.[approvalStep]?.suspend?.resume_form?.schema}
{#if items} -
-
-
{min ? displayDate(new Date(min), true) : ''}
{#if max && min} - {/if}
{max ? displayDate(new Date(max), true) : ''}{#if !max && min}{#if now} - {msToSec(now - min, 1)}s - {/if}{/if}
-
-
-
-
-
Waiting for executor/Suspend
-
+
+
+
+ {min ? displayDate(new Date(min), true) : ''} +
+
+
+
+ Wait
- -
-
Execution
-
+
+
+ Execution
+ {#if max && min} + {msToSec(max - min, 1)}s + {/if} + {#if !max && min}{#if now} + {msToSec(now - min, 1)}s + {/if}{/if}
{#if selfWaitTime} -
- root: +
+ root: x.created_at && x.started_at)} -
-
-
+
+
{k.startsWith('subflow:') ? k.substring(8) : k} {#if localModuleStates[k]?.selectedForloop && (typ == 'forloopflow' || typ == 'whileloopflow')} @@ -141,70 +136,67 @@ {/if}
-
- {#if subItems?.length > 1} -
- {subItems?.length} jobs -
- {/if} - {#if min && total} - subItems?.[index]?.id} - > - {#snippet item({ index, style })} - {@const b = subItems?.[index]} - {#if b?.created_at} - - {@const waitingLen = b?.created_at - ? b.started_at - ? b.started_at - b?.created_at - : b.duration_ms - ? 0 - : now - b?.created_at - : 0} -
+ {#if subItems?.length > 1} + + {subItems?.length} jobs + + {/if} +
+
+ {#if min && total} + subItems?.[index]?.id} + > + {#snippet item({ index, style })} + {@const b = subItems?.[index]} + {#if b?.created_at} + {@const waitingLen = b?.created_at + ? b.started_at + ? b.started_at - b?.created_at + : b.duration_ms + ? 0 + : now - b?.created_at + : 0} +
+ + {#if b.started_at} - {#if b.started_at} - - {/if} -
- {:else} -
-
- -
-
- {/if} - {/snippet} - - {/if}
+ {/if} +
+ {:else} +
+ {/if} + {/snippet} + + {/if} +
{/each}
+ {:else} {/if} diff --git a/frontend/src/lib/components/FlowViewer.svelte b/frontend/src/lib/components/FlowViewer.svelte index 2355ddf68d..1e321584e3 100644 --- a/frontend/src/lib/components/FlowViewer.svelte +++ b/frontend/src/lib/components/FlowViewer.svelte @@ -20,7 +20,7 @@ schema?: any } - type TabValue = 'ui' | 'raw' | 'schema' | 'diff' + export type TabValue = 'ui' | 'raw' | 'schema' | 'diff' interface Props { flow: { @@ -33,10 +33,16 @@ noSide?: boolean noGraph?: boolean initTab?: TabValue + selectedTab?: TabValue + hideTabs?: boolean noSummary?: boolean + noInput?: boolean + hideDefaultInputs?: boolean + showStepHint?: boolean noGraphDownload?: boolean availableVersions?: Array<{ id: number; deployment_msg?: string }> selectedVersionId?: number + graphContent?: import('svelte').Snippet } let { @@ -46,9 +52,15 @@ noGraph = false, availableVersions = undefined, initTab = undefined, + selectedTab = $bindable(), + hideTabs = false, noSummary = false, + noInput = false, + hideDefaultInputs = false, + showStepHint = false, noGraphDownload = false, - selectedVersionId = undefined + selectedVersionId = undefined, + graphContent = undefined }: Props = $props() let open: { [id: number]: boolean } = {} @@ -59,7 +71,10 @@ let previousVersionId: number | undefined = $state(undefined) let previousFlow: PreviousFlow | undefined = $state(undefined) - let tab: TabValue = $state(untrack(() => initTab) ?? 'diff') + const tabControlledExternally = selectedTab !== undefined + if (!tabControlledExternally) { + selectedTab = initTab ?? 'diff' + } let previousFlowCache: Record = {} @@ -90,16 +105,16 @@ }) $effect.pre(() => { - if (initTab) { + if (initTab || tabControlledExternally) { return } if (availableVersions && availableVersions.length > 0) { - tab = 'diff' + selectedTab = 'diff' } else { if (noGraph) { - tab = 'schema' + selectedTab = 'schema' } else { - tab = 'ui' + selectedTab = 'ui' } } }) @@ -127,7 +142,7 @@ - + {#if availableVersions && availableVersions.length > 0} {/if} @@ -167,23 +182,38 @@ {/if} -
- {#if !noSummary} -

{flow.summary}

-
{flow.description ?? ''}
- {/if} + {#if graphContent} + {@render graphContent()} + {:else} +
+ {#if showStepHint} +

Click on a step to see its details

+ {/if} + {#if !noSummary} +

{flow.summary}

+
{flow.description ?? ''}
+ {/if} -

- Flow Input -

- {#if flow.schema && flow.schema.properties && Object.keys(flow.schema.properties).length > 0 && flow.schema} - - {:else} -
No inputs
- {/if} + {#if !noInput} +

+ Flow Input +

+ {#if flow.schema && flow.schema.properties && Object.keys(flow.schema.properties).length > 0 && flow.schema} + + {:else} +
No inputs
+ {/if} + {/if} - -
+ +
+ {/if} diff --git a/frontend/src/lib/components/HighlightCode.svelte b/frontend/src/lib/components/HighlightCode.svelte index be6e6b6ef0..909b8daca2 100644 --- a/frontend/src/lib/components/HighlightCode.svelte +++ b/frontend/src/lib/components/HighlightCode.svelte @@ -14,6 +14,7 @@ import yaml from 'svelte-highlight/languages/yaml' import java from 'svelte-highlight/languages/java' import ruby from 'svelte-highlight/languages/ruby' + import r from 'svelte-highlight/languages/r' import type { Script } from '$lib/gen' import { Button } from './common' import { copyToClipboard } from '$lib/utils' @@ -91,6 +92,8 @@ return java case 'ruby': return ruby + case 'rlang': + return r case 'json': return json // for related places search: ADD_NEW_LANG diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 18514b3836..d558910aad 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -21,6 +21,7 @@ import SettingsFooter from './workspaceSettings/SettingsFooter.svelte' import SettingsPageHeader from './settings/SettingsPageHeader.svelte' import WorkspaceRegistries from './instanceSettings/WorkspaceRegistries.svelte' + import DbHealth from './instanceSettings/DbHealth.svelte' interface Props { tab?: string @@ -1052,6 +1053,12 @@ title="GitHub Enterprise App" description="Configure a self-managed GitHub App for GitHub Enterprise Server git sync." /> + {:else if category == 'DB Health'} + + {:else if category == 'Auth/OAuth/SAML'} { if (noLogs != lastNoLogs) { lastNoLogs = noLogs - if (!noLogs) { + if (!noLogs && !getActiveReplay()) { currentEventSource?.onerror?.(new Event(noLogsChangeRestartEvent)) const lastJobId = lastCompletedJobId if (lastJobId && (job || lastCallbacks?.loadExtraLogs)) { @@ -255,7 +255,7 @@ } } export async function getLogs() { - if (job) { + if (job && !getActiveReplay()) { refreshLogOffset() const getUpdate = await JobService.getJobUpdates({ workspace: workspace!, diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index c1a2a7ebdb..9bb4eeefc1 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -21,6 +21,7 @@ import Skeleton from './common/skeleton/Skeleton.svelte' import Button from './common/button/Button.svelte' import { sameTopDomainOrigin } from '$lib/cookies' + import { isValidLogoutRedirect } from '$lib/logoutRedirect' interface Props { rd?: string | undefined @@ -134,7 +135,11 @@ async function redirectUser() { if (rd?.startsWith('http')) { - window.location.href = rd + if (isValidLogoutRedirect(rd)) { + window.location.href = rd + return + } + goto('/') return } if ($workspaceStore) { diff --git a/frontend/src/lib/components/OktaSetting.svelte b/frontend/src/lib/components/OktaSetting.svelte index 8f19ce5666..495ed5794c 100644 --- a/frontend/src/lib/components/OktaSetting.svelte +++ b/frontend/src/lib/components/OktaSetting.svelte @@ -18,17 +18,18 @@ function changeDomain(domain, custom) { if (value) { let baseUrl = custom ? `https://${domain}` : `https://${domain}.okta.com` + let authPath = custom ? '/v1' : '/oauth2/v1' value = { ...value, login_config: { - auth_url: `${baseUrl}/oauth2/v1/authorize`, - token_url: `${baseUrl}/oauth2/v1/token`, - userinfo_url: `${baseUrl}/oauth2/v1/userinfo`, + auth_url: `${baseUrl}${authPath}/authorize`, + token_url: `${baseUrl}${authPath}/token`, + userinfo_url: `${baseUrl}${authPath}/userinfo`, scopes: ['openid', 'profile', 'email'] }, connect_config: { - auth_url: `${baseUrl}/oauth2/v1/authorize`, - token_url: `${baseUrl}/oauth2/v1/token`, + auth_url: `${baseUrl}${authPath}/authorize`, + token_url: `${baseUrl}${authPath}/token`, scopes: ['openid', 'profile', 'email'] } } diff --git a/frontend/src/lib/components/Password.svelte b/frontend/src/lib/components/Password.svelte index 673f6416db..12c374bf70 100644 --- a/frontend/src/lib/components/Password.svelte +++ b/frontend/src/lib/components/Password.svelte @@ -1,5 +1,6 @@
-
+
- onBlur?.(e), - onkeydown: (e) => { - onKeyDown?.(e) - bubble('keydown')(e) - }, - type: hideValue ? 'password' : 'text' - }} - class="pr-8" - /> + {#if isMultiline} + onBlur?.(e), + onkeydown: (e) => { + onKeyDown?.(e) + bubble('keydown')(e) + }, + style: hideValue ? '-webkit-text-security: disc' : '' + }} + class="pr-8" + unifiedHeight={false} + /> + {:else} + onBlur?.(e), + onkeydown: (e) => { + if (e.key === 'Enter') { + e.preventDefault() + insertAndSwitchToMultiline(e.currentTarget as HTMLInputElement, '\n') + return + } + onKeyDown?.(e) + bubble('keydown')(e) + }, + onpaste: (e) => { + const text = e.clipboardData?.getData('text') + if (text?.includes('\n')) { + e.preventDefault() + insertAndSwitchToMultiline(e.currentTarget as HTMLInputElement, text) + } + }, + type: hideValue ? 'password' : 'text' + }} + class="pr-8" + /> + {/if}
{#if red}
This field is required
diff --git a/frontend/src/lib/components/PasswordArgInput.svelte b/frontend/src/lib/components/PasswordArgInput.svelte index 8e7d548c1b..60474e53cb 100644 --- a/frontend/src/lib/components/PasswordArgInput.svelte +++ b/frontend/src/lib/components/PasswordArgInput.svelte @@ -9,9 +9,10 @@ interface Props { value?: string | undefined disabled: boolean + minRows?: number } - let { value = $bindable(undefined), disabled }: Props = $props() + let { value = $bindable(undefined), disabled, minRows }: Props = $props() let path = $state('') let password = $state( @@ -95,5 +96,5 @@
{:else} - + {/if} diff --git a/frontend/src/lib/components/Range.svelte b/frontend/src/lib/components/Range.svelte index 72129aada2..736685ae74 100644 --- a/frontend/src/lib/components/Range.svelte +++ b/frontend/src/lib/components/Range.svelte @@ -19,7 +19,7 @@ min = 0, max = 100, initialValue = 0, - value = $bindable(typeof initialValue === 'string' ? parseInt(initialValue) : initialValue), + value = $bindable(), disabled = false, defaultValue = undefined, format = (v) => `${v}`, @@ -36,8 +36,14 @@ } run(() => { - if (value === null) { - value = 0 + if (value === null || value === undefined || Number.isNaN(value)) { + const fallback = + initialValue !== undefined + ? typeof initialValue === 'string' + ? parseInt(initialValue) + : initialValue + : (min ?? 0) + value = Number.isNaN(fallback) ? (min ?? 0) : fallback } }) diff --git a/frontend/src/lib/components/RunForm.svelte b/frontend/src/lib/components/RunForm.svelte index da0c87a15f..6c2ac09da0 100644 --- a/frontend/src/lib/components/RunForm.svelte +++ b/frontend/src/lib/components/RunForm.svelte @@ -3,7 +3,8 @@ computeSharableHash as computeSharableHash, defaultIfEmptyString, emptyString, - truncateHash + truncateHash, + sendUserToast } from '$lib/utils' import type { Schema } from '$lib/common' @@ -21,6 +22,7 @@ import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte' import InputSelectedBadge from './schema/InputSelectedBadge.svelte' import { untrack } from 'svelte' + import { processSecretArgs } from './secretArgUtils' let reloadArgs = $state(0) let jsonEditor: JsonInputs | undefined = $state(undefined) @@ -33,8 +35,20 @@ reloadArgs++ } - export function run() { - runAction(scheduledForStr, args ?? {}, invisible_to_owner, overrideTag) + export async function run(overrideScheduledForStr?: string | undefined | null) { + let processedArgs: Record + try { + processedArgs = await processSecretArgs(args ?? {}, runnable?.schema) + } catch (e) { + sendUserToast('Failed to process sensitive args: ' + e, true) + return + } + runAction( + overrideScheduledForStr === null ? undefined : (overrideScheduledForStr ?? scheduledForStr), + processedArgs, + invisible_to_owner, + overrideTag + ) } interface Props { @@ -276,7 +290,7 @@ unifiedSize="md" btnClasses="!inline-flex" disabled={!isValid && !jsonView} - on:click={() => runAction(scheduledForStr, args ?? {}, invisible_to_owner, overrideTag)} + on:click={() => run()} shortCut={{ Icon: CornerDownLeft, hide: !viewKeybinding }} > {scheduledForStr ? 'Schedule to run later' : buttonText} @@ -315,7 +329,7 @@ btnClasses="!px-6 !py-1 w-full" variant="accent" disabled={!isValid && !jsonView} - on:click={() => runAction(undefined, args ?? {}, invisible_to_owner, overrideTag)} + on:click={() => run(null)} shortCut={{ Icon: CornerDownLeft, hide: !viewKeybinding }} > {buttonText} diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 024fe0e907..d55fe12146 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -178,7 +178,7 @@ let confirmCallback: () => void = $state(() => {}) // What happens when user clicks `override` in warning let open: boolean = $state(false) // Is confirmation modal open let args: Record = $state(untrack(() => initialArgs)) // Test args input - let selectedInputTab: 'main' | 'preprocessor' = $state('main') + let selectedInputTab: 'main' | 'preprocessor' | 'diagram' = $state('main') let hasPreprocessor = $state(false) let preserveOnBehalfOf = $state(false) @@ -1268,7 +1268,7 @@ } as ButtonType.Icon} > {label} - {#if lang === 'ruby'} + {#if lang === 'rlang'} BETA {/if} diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index 2690301f28..66101fc892 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -1,5 +1,6 @@ - + diff --git a/frontend/src/lib/components/StringTypeNarrowing.svelte b/frontend/src/lib/components/StringTypeNarrowing.svelte index 7d84b4c4fc..8ecedf039e 100644 --- a/frontend/src/lib/components/StringTypeNarrowing.svelte +++ b/frontend/src/lib/components/StringTypeNarrowing.svelte @@ -52,7 +52,8 @@ computeKind(enum_, contentEncoding, pattern, format) ) - const allowKindChange = untrack(() => overrideAllowKindChange) || untrack(() => originalType) === 'string' + const allowKindChange = + untrack(() => overrideAllowKindChange) || untrack(() => originalType) === 'string' let patternStr: string = $state(pattern ?? '') let resource: string | undefined = $state() @@ -383,7 +384,7 @@ options={{ right: 'Is Password/Sensitive', rightTooltip: - 'The value will be stored as an ephemeral secret variable in the user space of the caller of the job, only viewable by him.' + 'The value will be stored as an ephemeral secret variable in the user space of the caller of the job, only viewable by that user.' }} checked={password} on:change={(e) => { diff --git a/frontend/src/lib/components/TimelineBar.svelte b/frontend/src/lib/components/TimelineBar.svelte index e4fb511450..5d1f53e961 100644 --- a/frontend/src/lib/components/TimelineBar.svelte +++ b/frontend/src/lib/components/TimelineBar.svelte @@ -14,6 +14,7 @@ running: boolean concat?: boolean gray?: boolean + spacerClass?: string } let { @@ -25,25 +26,26 @@ id, running, concat = false, - gray = false + gray = false, + spacerClass = '' }: Props = $props() {#if min && started_at != undefined} {#if !concat} -
+
{/if} {#snippet text()} 0} {@const narrow = len / total < 0.09} - {@const endPos = started_at != undefined && min != undefined ? (started_at - min + len) / total : 1} + {@const endPos = + started_at != undefined && min != undefined ? (started_at - min + len) / total : 1} {@const nearStart = endPos < 0.15} - {#if len}{msToSec(len, 1)}s{/if} {/if} diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte index 5d6ddf7b93..6c991ea52a 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte @@ -4,9 +4,11 @@ const bubble = createBubbler() import { Button, Drawer, DrawerContent } from '$lib/components/common' import { base } from '$lib/base' + import FlowGraphViewer from '$lib/components/FlowGraphViewer.svelte' + import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import FlowModuleScript from '$lib/components/flows/content/FlowModuleScript.svelte' import FlowPathViewer from '$lib/components/flows/content/FlowPathViewer.svelte' - import { emptySchema, sendUserToast } from '$lib/utils' + import { emptySchema, getHubFlowIdFromPath, isHubFlowPath, sendUserToast } from '$lib/utils' import { getContext, tick, untrack } from 'svelte' import type { ConnectedAppInput, @@ -31,7 +33,8 @@ import Popover from '$lib/components/meltComponents/Popover.svelte' import ScriptEditorDrawer from '$lib/components/flows/content/ScriptEditorDrawer.svelte' import FlowEditorDrawer from '$lib/components/flows/content/FlowEditorDrawer.svelte' - import { ScriptService } from '$lib/gen' + import { FlowService, ScriptService, type OpenFlow } from '$lib/gen' + import { replaceScriptPlaceholderWithItsValues } from '$lib/hub' interface Props { runnable: RunnableByPath @@ -43,6 +46,7 @@ isLoading?: boolean onRun?: any onCancel?: any + hubFlowPreview?: OpenFlow | undefined } let { @@ -52,14 +56,17 @@ rawApps = false, isLoading = false, onRun = async () => {}, - onCancel = async () => {} + onCancel = async () => {}, + hubFlowPreview = $bindable(undefined) }: Props = $props() const viewerContext = getContext('AppViewerContext') let drawerFlowViewer: Drawer | undefined = $state(undefined) let flowPath: string = $state('') + let drawerShowsHubFlow = $state(false) let notFound = $state(false) + let hubFlowId = $derived(getHubFlowIdFromPath(runnable.path)) // Key to force re-mounting of viewer components (bypasses FlowModuleScript cache) let refreshKey = $state(0) @@ -70,6 +77,7 @@ const dispatch = createEventDispatcher() async function refreshScript(runnable: RunnableByPath) { + hubFlowPreview = undefined try { let { schema } = await getScriptByPath(runnable.path) if (!deepEqual(runnable.schema, schema)) { @@ -86,7 +94,39 @@ } async function refreshFlow(runnable: RunnableByPath) { + hubFlowPreview = undefined try { + const hubFlowId = getHubFlowIdFromPath(runnable.path) + if (hubFlowId !== undefined) { + const hub = await FlowService.getHubFlowById({ id: hubFlowId }) + const flow = hub.flow ? structuredClone(hub.flow) : undefined + if (flow?.value.preprocessor_module?.value.type === 'rawscript') { + flow.value.preprocessor_module.value.content = replaceScriptPlaceholderWithItsValues( + String(hubFlowId), + flow.value.preprocessor_module.value.content + ) + } + + if (!flow) { + notFound = true + return + } + + hubFlowPreview = flow + const schema = + flow.schema && typeof flow.schema === 'object' && Object.keys(flow.schema).length > 0 + ? (flow.schema as any) + : emptySchema() + if (!deepEqual(runnable.schema, schema)) { + runnable.schema = schema + if (!runnable.schema.order) { + runnable.schema.order = Object.keys(runnable.schema.properties ?? {}) + } + fields = computeFields(schema, false, fields ?? {}) + } + return + } + const { schema } = (await loadSchema($workspaceStore ?? '', runnable.path, 'flow')) ?? emptySchema() if (!deepEqual(runnable.schema, schema)) { @@ -158,6 +198,8 @@ refreshScript(runnable) } else if (runnable.runType == 'flow') { refreshFlow(runnable) + } else { + hubFlowPreview = undefined } lastRunnable = runnable } @@ -170,8 +212,34 @@ - - + { + flowPath = '' + drawerShowsHubFlow = false + drawerFlowViewer?.closeDrawer() + }} + > + {#if drawerShowsHubFlow} +
+ {#if hubFlowPreview} + + {:else if notFound} +
Hub flow not found at {flowPath}
+ {:else} +
+ +
+ {/if} +
+ {:else if flowPath} + + {/if} @@ -210,7 +278,7 @@ size="xs" startIcon={{ icon: RefreshCw }} on:click={async () => { - sendUserToast('Getting latest script version at that path') + sendUserToast('Getting latest runnable version at that path') // Increment refreshKey to force re-mounting of viewer components (bypasses cache) refreshKey++ lastRunnable = undefined @@ -238,31 +306,45 @@ startIcon={{ icon: Eye }} on:click={() => { flowPath = runnable.path + drawerShowsHubFlow = isHubFlowPath(runnable.path) drawerFlowViewer?.openDrawer() }} > Expand - - + {#if hubFlowId} + + {:else} + + + {/if} {:else} + Cache + {/snippet} {#snippet content()} - Since this is a reference to a workspace {runnable.runType}, set the cache in the {runnable.runType} - settings directly by editing it. The cache will be shared by any app or flow that uses this - {runnable.runType}. + {#if runnable.runType == 'flow' && isHubFlowPath(runnable.path)} + Since this is a reference to a hub flow, cache settings are managed from the flow after + you fork it into your workspace. + {:else} + Since this is a reference to a workspace {runnable.runType}, set the cache in the + {runnable.runType} settings directly by editing it. The cache will be shared by any app or + flow that uses this {runnable.runType}. + {/if} {/snippet} @@ -325,18 +414,37 @@ class="!text-xs !rounded-xs" />
-
+
{#key `${viewerContext?.stateId ? get(viewerContext.stateId) : 0}-${refreshKey}`} {#if notFound} -
{runnable.runType} not found at {runnable.path} in workspace {$workspaceStore}
+
+ {#if runnable.runType == 'flow' && isHubFlowPath(runnable.path)} + Hub flow not found at {runnable.path} + {:else} + {runnable.runType} not found at {runnable.path} in workspace {$workspaceStore} + {/if} +
{:else if runnable.runType == 'script' || runnable.runType == 'hubscript'}
{:else if runnable.runType == 'flow'} - + {#if isHubFlowPath(runnable.path)} + {#if hubFlowPreview} +
+ +
+ {:else} + + {/if} + {:else} + + {/if} {:else} Unrecognized runType {runnable.runType} {/if} diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte index 23419e51be..717f59ac6e 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte @@ -8,10 +8,10 @@ import WorkspaceFlowList from './WorkspaceFlowList.svelte' import { createEventDispatcher, untrack } from 'svelte' import type { Schema } from '$lib/common' - import { schemaToInputsSpec } from '$lib/components/apps/utils' - import { defaultIfEmptyString, emptySchema } from '$lib/utils' + import { emptySchema } from '$lib/utils' import { loadSchema } from '$lib/infer' import { workspaceStore } from '$lib/stores' + import { buildPathRunnableSelection } from './runnableSelectorUtils' type TabType = 'hubscripts' | 'workspacescripts' | 'workspaceflows' | 'inlinescripts' @@ -62,51 +62,44 @@ } async function pickScript(path: string) { - const schema = await loadSchemaFromTriggerable(path, 'script') - const fields = schemaToInputsSpec(schema.schema, defaultUserInput) - const runnable = { - type: 'path', + const selection = buildPathRunnableSelection( path, - runType: 'script', - schema: schema.schema, - name: defaultIfEmptyString(schema.summary, path) - } as const - + 'script', + await loadSchemaFromTriggerable(path, 'script'), + defaultUserInput, + rawApps + ) dispatch('pick', { - runnable, - fields + runnable: selection.runnable, + fields: selection.fields }) } async function pickFlow(path: string) { - const schema = await loadSchemaFromTriggerable(path, 'flow') - const fields = schemaToInputsSpec(schema.schema, defaultUserInput) - const runnable = { - type: 'path', + const selection = buildPathRunnableSelection( path, - runType: 'flow', - schema, - name: defaultIfEmptyString(schema.summary, path) - } as const + 'flow', + await loadSchemaFromTriggerable(path, 'flow'), + defaultUserInput, + rawApps + ) dispatch('pick', { - runnable, - fields + runnable: selection.runnable, + fields: selection.fields }) } async function pickHubScript(path: string) { - const schema = await loadSchemaFromTriggerable(path, 'hubscript') - const fields = schemaToInputsSpec(schema.schema, defaultUserInput) - const runnable = { - type: 'path', + const selection = buildPathRunnableSelection( path, - runType: 'hubscript', - schema: schema.schema, - name: defaultIfEmptyString(schema.summary, path) - } as const + 'hubscript', + await loadSchemaFromTriggerable(path, 'hubscript'), + defaultUserInput, + rawApps + ) dispatch('pick', { - runnable, - fields + runnable: selection.runnable, + fields: selection.fields }) } diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/runnableSelectorUtils.test.ts b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/runnableSelectorUtils.test.ts new file mode 100644 index 0000000000..8968363f03 --- /dev/null +++ b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/runnableSelectorUtils.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest' + +import type { LoadedRunnableSchema } from './runnableSelectorUtils' +import { buildPathRunnableSelection } from './runnableSelectorUtils' + +const loadedSchema: LoadedRunnableSchema = { + summary: 'My flow', + schema: { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + required: ['string_input'], + properties: { + string_input: { + type: 'string', + default: '' + } + } + } +} + +describe('buildPathRunnableSelection', () => { + it('keeps the actual schema object and defaults raw-app fields to user mode', () => { + const selection = buildPathRunnableSelection( + 'u/dev/my_flow', + 'flow', + loadedSchema, + false, + true + ) + + expect(selection.runnable).toMatchObject({ + type: 'path', + path: 'u/dev/my_flow', + runType: 'flow', + schema: loadedSchema.schema, + name: 'My flow' + }) + expect(selection.fields.string_input.type).toBe('user') + expect(selection.fields.string_input.value).toBe('') + }) + + it('preserves static defaults for non-raw-app pickers', () => { + const selection = buildPathRunnableSelection( + 'u/dev/my_flow', + 'flow', + loadedSchema, + false, + false + ) + + expect(selection.fields.string_input.type).toBe('static') + }) +}) diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/runnableSelectorUtils.ts b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/runnableSelectorUtils.ts new file mode 100644 index 0000000000..44e989e925 --- /dev/null +++ b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/runnableSelectorUtils.ts @@ -0,0 +1,31 @@ +import type { Schema } from '$lib/common' +import type { Runnable, StaticAppInput } from '$lib/components/apps/inputType' +import { schemaToInputsSpec } from '$lib/components/apps/utils' +import { defaultIfEmptyString } from '$lib/utils' + +export type LoadedRunnableSchema = { + schema: Schema + summary: string | undefined +} + +export function buildPathRunnableSelection( + path: string, + runType: 'script' | 'flow' | 'hubscript', + loadedSchema: LoadedRunnableSchema, + defaultUserInput: boolean, + rawApps: boolean +): { + runnable: Runnable + fields: Record +} { + return { + runnable: { + type: 'path', + path, + runType, + schema: loadedSchema.schema, + name: defaultIfEmptyString(loadedSchema.summary, path) + }, + fields: schemaToInputsSpec(loadedSchema.schema, defaultUserInput || rawApps) + } +} diff --git a/frontend/src/lib/components/assets/AssetButtons.svelte b/frontend/src/lib/components/assets/AssetButtons.svelte index 62bd981609..8e8c851f42 100644 --- a/frontend/src/lib/components/assets/AssetButtons.svelte +++ b/frontend/src/lib/components/assets/AssetButtons.svelte @@ -43,32 +43,28 @@ {#if (asset.kind === 'resource' && resourceDataCacheValue === undefined) || ducklakeNotFound || datatableNotFound} {#snippet trigger()} - - - {:else if datatableNotFound} - - {:else if asset.kind === 'resource' && resourceDataCacheValue === undefined} - - {/if} - - {/snippet} + Not found + {#if ducklakeNotFound} + + {:else if datatableNotFound} + + {:else if asset.kind === 'resource' && resourceDataCacheValue === undefined} + + {/if} + {/snippet} {:else if assetCanBeExplored(asset, { resource_type: resourceDataCacheValue })} -{#snippet rightBadge(text: string | undefined, tooltip?: string)} - {#if text} +{#snippet rightBadge(badgeText: string | undefined, tooltip?: string)} + {#if badgeText}
- {text} + {badgeText}
{#snippet text()} - - {#if tooltip} - {tooltip} - {/if} - - {/snippet} + {#if tooltip} + {tooltip} + {/if} + {/snippet} {/if} {/snippet} diff --git a/frontend/src/lib/components/common/fileUpload/FileUpload.svelte b/frontend/src/lib/components/common/fileUpload/FileUpload.svelte index a7ba9a1c34..dfe5c158cc 100644 --- a/frontend/src/lib/components/common/fileUpload/FileUpload.svelte +++ b/frontend/src/lib/components/common/fileUpload/FileUpload.svelte @@ -6,6 +6,7 @@ import { sendUserToast } from '$lib/toast' import { workspaceStore } from '$lib/stores' import { AppService, HelpersService } from '$lib/gen' + import { OpenAPI } from '$lib/gen/core/OpenAPI' import { writable, type Writable } from 'svelte/store' import { Ban, CheckCheck, FileWarning, Files, RefreshCcw, Trash, XIcon } from 'lucide-svelte' import { twMerge } from 'tailwind-merge' @@ -334,6 +335,9 @@ true ) xhr?.setRequestHeader('Content-Type', 'application/octet-stream') + if (OpenAPI.TOKEN) { + xhr?.setRequestHeader('Authorization', `Bearer ${OpenAPI.TOKEN}`) + } xhr?.send(fileToUpload) })) as any diff --git a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte index ca3900eb9f..09ff1082e6 100644 --- a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte +++ b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte @@ -25,6 +25,7 @@ import JavaIcon from '$lib/components/icons/JavaIcon.svelte' import DuckDbIcon from '$lib/components/icons/DuckDbIcon.svelte' import RubyIcon from '$lib/components/icons/RubyIcon.svelte' + import RIcon from '$lib/components/icons/RIcon.svelte' import ClaudeIcon from '$lib/components/icons/ClaudeIcon.svelte' interface Props { @@ -72,6 +73,7 @@ nu: 'Nu', java: 'Java', ruby: 'Ruby', + rlang: 'R', claudesandbox: 'Claude Sandbox' // for related places search: ADD_NEW_LANG } @@ -107,6 +109,7 @@ nu: NuIcon, java: JavaIcon, ruby: RubyIcon, + rlang: RIcon, duckdb: DuckDbIcon, claudesandbox: TypeScriptIcon // for related places search: ADD_NEW_LANG diff --git a/frontend/src/lib/components/custom_ui.ts b/frontend/src/lib/components/custom_ui.ts index 536471f74d..5f017ee950 100644 --- a/frontend/src/lib/components/custom_ui.ts +++ b/frontend/src/lib/components/custom_ui.ts @@ -42,6 +42,8 @@ export type FlowBuilderWhitelabelCustomUi = { tagLabel?: string aiAgent?: boolean aiSandbox?: boolean + suggestIntegration?: boolean + suggestScript?: boolean } export type DisplayResultUi = { diff --git a/frontend/src/lib/components/flows/content/FlowInput.svelte b/frontend/src/lib/components/flows/content/FlowInput.svelte index a9eab8074c..5c7002f147 100644 --- a/frontend/src/lib/components/flows/content/FlowInput.svelte +++ b/frontend/src/lib/components/flows/content/FlowInput.svelte @@ -686,6 +686,7 @@ bind:schema={flowStore.val.schema} hiddenArgs={['user_message']} isFlowInput + showSensitiveToggle editTab={chatInputsEditTab ? 'inputEditor' : undefined} showDynOpt bind:dynCode @@ -741,6 +742,7 @@ bind:this={editableSchemaForm} bind:schema={flowStore.val.schema} isFlowInput + showSensitiveToggle on:delete={(e) => { addPropertyV2?.handleDeleteArgument([e.detail]) }} diff --git a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte index 0dd5635816..db199c745a 100644 --- a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte @@ -308,7 +308,7 @@ bind:selectedFilter={selected} resourceType /> - {#if !selected} + {#if !selected && customUi?.suggestIntegration != false}
{#if flow} - + {:else} {/if} diff --git a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte index 3d929dead7..0c548d34c9 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte @@ -36,7 +36,7 @@ + +
+ {#if errors.length > 0} +
+ {#each errors as error (error.line)} +
+ + + {#if error.line > 0} + L{error.line}: + {/if} + {error.message} + +
+ {/each} +
+ {:else if empty} +
+ + No workflow diagram +
+ {:else} + + + + + + {/if} +
diff --git a/frontend/src/lib/components/graph/noteEditor.svelte.ts b/frontend/src/lib/components/graph/noteEditor.svelte.ts index e59e3be4f5..50c2df7bfa 100644 --- a/frontend/src/lib/components/graph/noteEditor.svelte.ts +++ b/frontend/src/lib/components/graph/noteEditor.svelte.ts @@ -219,7 +219,10 @@ export class NoteEditor { /** * Clean up group notes using DAG path completion */ - cleanupGroupNotes(flowNodes: { id: string; parentIds?: string[] }[]): void { + cleanupGroupNotes( + flowNodes: { id: string; parentIds?: string[] }[], + collapsedModuleIds?: Set + ): void { if (!this.isAvailable()) { return } @@ -231,6 +234,13 @@ export class NoteEditor { let hasChanges = false const nodeSet = new Set(flowNodes.map((n) => n.id)) + // Include collapsed module IDs as valid — they are hidden but still exist + if (collapsedModuleIds) { + for (const id of collapsedModuleIds) { + nodeSet.add(id) + } + } + // Step 1: Clean invalid nodes from existing group notes for (const note of groupNotes) { const originalIds = note.contained_node_ids || [] @@ -249,6 +259,12 @@ export class NoteEditor { const originalNodes = note.contained_node_ids || [] if (originalNodes.length === 0) continue + // Skip path completion for notes that reference collapsed modules, + // since the DAG is incomplete when groups are collapsed + if (collapsedModuleIds && originalNodes.some((id) => collapsedModuleIds.has(id))) { + continue + } + // Use the DAG path completion and splitting algorithm const completedGroups = completeAndSplitGroup(originalNodes, flowNodes) diff --git a/frontend/src/lib/components/graph/noteUtils.svelte.ts b/frontend/src/lib/components/graph/noteUtils.svelte.ts index bd7fc2ce05..577b421a9e 100644 --- a/frontend/src/lib/components/graph/noteUtils.svelte.ts +++ b/frontend/src/lib/components/graph/noteUtils.svelte.ts @@ -249,7 +249,8 @@ export function computeNoteNodes( noteTextHeights: Record, onTextHeightChange: (noteId: string, height: number) => void, editMode: boolean = false, - noteEditorContext: NoteEditorContext | undefined + noteEditorContext: NoteEditorContext | undefined, + collapsedModuleIds?: Set ): NoteComputeResult { // Check cache first if ( @@ -263,7 +264,7 @@ export function computeNoteNodes( if (editMode) { if (noteEditorContext?.noteEditor?.isAvailable()) { - noteEditorContext.noteEditor.cleanupGroupNotes(nodes) + noteEditorContext.noteEditor.cleanupGroupNotes(nodes, collapsedModuleIds) } } @@ -290,6 +291,15 @@ export function computeNoteNodes( for (const note of notes) { const isGroupNote = note.type === 'group' + + // Skip group notes whose contained nodes are all inside collapsed groups + if (isGroupNote && collapsedModuleIds?.size) { + const ids = note.contained_node_ids ?? [] + if (ids.length > 0 && ids.every((id) => collapsedModuleIds.has(id))) { + continue + } + } + const zIndex = noteZIndexes[note.id] // Calculate position and size using node positions for group notes diff --git a/frontend/src/lib/components/graph/renderers/edges/WacEdge.svelte b/frontend/src/lib/components/graph/renderers/edges/WacEdge.svelte new file mode 100644 index 0000000000..6f52a28e11 --- /dev/null +++ b/frontend/src/lib/components/graph/renderers/edges/WacEdge.svelte @@ -0,0 +1,57 @@ + + + + +{#if label} + + {label} + +{/if} + +{#if animated} + +{/if} + + diff --git a/frontend/src/lib/components/graph/renderers/nodes/WacControlNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/WacControlNode.svelte new file mode 100644 index 0000000000..0da9269d7d --- /dev/null +++ b/frontend/src/lib/components/graph/renderers/nodes/WacControlNode.svelte @@ -0,0 +1,57 @@ + + +
+
+
+
{displayLabel}
+
+
+
+ + + diff --git a/frontend/src/lib/components/graph/renderers/nodes/WacStepNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/WacStepNode.svelte new file mode 100644 index 0000000000..18e77ac946 --- /dev/null +++ b/frontend/src/lib/components/graph/renderers/nodes/WacStepNode.svelte @@ -0,0 +1,39 @@ + + +
+
+
+
+
{label}
+
+ {#if hasExternalPath} + + {script} + + {:else if isInline} + + inline + + {/if} +
+
+
+ + + diff --git a/frontend/src/lib/components/graph/wacDagLayout.ts b/frontend/src/lib/components/graph/wacDagLayout.ts new file mode 100644 index 0000000000..9f1d02f020 --- /dev/null +++ b/frontend/src/lib/components/graph/wacDagLayout.ts @@ -0,0 +1,139 @@ +import type { Node, Edge } from '@xyflow/svelte' +import type { WacWorkflowDag, WacDagNode } from '$lib/infer' +import { NODE } from './util' + +const GAP_X = NODE.gap.horizontal +const GAP_Y = NODE.gap.vertical + +/** + * Simple top-to-bottom DAG layout for WAC workflow graphs. + * Uses the same NODE dimensions as the flow editor for visual consistency. + */ +export function dagToXyflow(dag: WacWorkflowDag): { nodes: Node[]; edges: Edge[] } { + if (dag.nodes.length === 0) { + return { nodes: [], edges: [] } + } + + // Build adjacency maps + const childrenMap = new Map() + const parentMap = new Map() + + for (const edge of dag.edges) { + if (!childrenMap.has(edge.from)) childrenMap.set(edge.from, []) + childrenMap.get(edge.from)!.push({ id: edge.to, label: edge.label }) + if (!parentMap.has(edge.to)) parentMap.set(edge.to, []) + parentMap.get(edge.to)!.push(edge.from) + } + + // Find root nodes (no parents, excluding back-edges to loop starts) + const roots = dag.nodes.filter((n) => { + const parents = parentMap.get(n.id) ?? [] + return ( + parents.length === 0 || + parents.every((p) => { + const edge = dag.edges.find((e) => e.from === p && e.to === n.id) + return edge?.label === 'next' + }) + ) + }) + + // Assign layers using BFS (ignoring back-edges) + const layers = new Map() + const queue: string[] = [] + + for (const root of roots) { + layers.set(root.id, 0) + queue.push(root.id) + } + + while (queue.length > 0) { + const nodeId = queue.shift()! + const layer = layers.get(nodeId)! + const children = childrenMap.get(nodeId) ?? [] + + for (const child of children) { + if (child.label === 'next') continue // skip back-edges + const existing = layers.get(child.id) + if (existing === undefined || existing < layer + 1) { + layers.set(child.id, layer + 1) + queue.push(child.id) + } + } + } + + // Group nodes by layer + const layerGroups = new Map() + for (const [nodeId, layer] of layers) { + if (!layerGroups.has(layer)) layerGroups.set(layer, []) + layerGroups.get(layer)!.push(nodeId) + } + + const maxLayer = Math.max(...layers.values(), 0) + + // Position nodes — centered, using flow editor dimensions + const positions = new Map() + + for (let layer = 0; layer <= maxLayer; layer++) { + const group = layerGroups.get(layer) ?? [] + const totalWidth = group.length * NODE.width + (group.length - 1) * GAP_X + const startX = -totalWidth / 2 + + for (let i = 0; i < group.length; i++) { + positions.set(group[i], { + x: startX + i * (NODE.width + GAP_X), + y: layer * (NODE.height + GAP_Y) + }) + } + } + + // Convert to xyflow nodes + const nodeMap = new Map(dag.nodes.map((n) => [n.id, n])) + const xyNodes: Node[] = [] + + for (const [id, pos] of positions) { + const dagNode = nodeMap.get(id) + if (!dagNode) continue + + xyNodes.push({ + id, + type: getXyflowNodeType(dagNode), + position: { x: pos.x, y: pos.y }, + data: { dagNode }, + width: NODE.width, + height: NODE.height + }) + } + + // Convert to xyflow edges + const xyEdges: Edge[] = dag.edges.map((e, i) => ({ + id: `e-${i}`, + source: e.from, + target: e.to, + type: 'wacEdge', + label: e.label === 'next' ? '' : (e.label ?? ''), + animated: e.label === 'next', + style: e.label === 'next' ? 'stroke-dasharray: 5 5;' : undefined + })) + + return { nodes: xyNodes, edges: xyEdges } +} + +function getXyflowNodeType(node: WacDagNode): string { + switch (node.node_type.type) { + case 'Step': + case 'InlineStep': + return 'wacStep' + case 'Sleep': + case 'WaitForApproval': + case 'Branch': + case 'ParallelStart': + case 'ParallelEnd': + case 'LoopStart': + case 'LoopEnd': + case 'Merge': + case 'Return': + return 'wacControl' + default: + return 'wacStep' + } +} diff --git a/frontend/src/lib/components/graph/wacToFlow.ts b/frontend/src/lib/components/graph/wacToFlow.ts index 91d335ff91..6f2f3720e1 100644 --- a/frontend/src/lib/components/graph/wacToFlow.ts +++ b/frontend/src/lib/components/graph/wacToFlow.ts @@ -9,7 +9,7 @@ export function isWorkflowAsCode(code: string, language: string): boolean { return ( /workflow\s*\(/.test(code) && /task\s*\(/.test(code) && - /import.*(?:workflow|task).*from\s+['"]windmill-client(?:@[^'"]*)?['"]/.test(code) + /['"]windmill-client(?:@[^'"]*)?['"]/.test(code) ) } return false diff --git a/frontend/src/lib/components/icons/RIcon.svelte b/frontend/src/lib/components/icons/RIcon.svelte new file mode 100644 index 0000000000..388193fc2d --- /dev/null +++ b/frontend/src/lib/components/icons/RIcon.svelte @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index ca6b702795..585b29ea37 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -376,6 +376,7 @@ export const settings: Record = { } ], 'Auth/OAuth/SAML': [], + 'DB Health': [], Registries: [ { label: 'Instance Python Version', @@ -829,6 +830,12 @@ export const instanceSettingsNavigationGroups = [ aiId: 'instance-settings-indexer', aiDescription: 'Instance indexer settings', isEE: true + }, + { + id: 'db_health', + label: 'DB Health', + aiId: 'instance-settings-db-health', + aiDescription: 'Database health diagnostics and performance insights' } ] }, @@ -900,7 +907,8 @@ export const tabToCategoryMap: Record = { jobs: 'Jobs', private_hub: 'Private Hub', github_enterprise_app: 'GitHub App', - websocket: 'WebSocket' + websocket: 'WebSocket', + db_health: 'DB Health' } export const tabToAuthSubTab: Record = { @@ -933,7 +941,8 @@ export const categoryToTabMap: Record = { Jobs: 'jobs', 'Private Hub': 'private_hub', 'GitHub App': 'github_enterprise_app', - WebSocket: 'websocket' + WebSocket: 'websocket', + 'DB Health': 'db_health' } export interface SearchableSettingItem { diff --git a/frontend/src/lib/components/instanceSettings/DbHealth.svelte b/frontend/src/lib/components/instanceSettings/DbHealth.svelte new file mode 100644 index 0000000000..9614422552 --- /dev/null +++ b/frontend/src/lib/components/instanceSettings/DbHealth.svelte @@ -0,0 +1,510 @@ + + +
+
+ + + {#if loading} + This may take a few seconds... + {/if} +
+ + {#if error} +
+ {error} +
+ {/if} + + {#if data} + +
+ + {#if expandedSections.database_size} +
+

+ Total database size: {data.database_size.total_size_pretty} +

+
+ + + + + + + + + {#each data.database_size.top_tables as t} + + + + + {/each} + +
TableSize
{t.table_name}{t.total_size_pretty}
+
+
+ {/if} +
+ + +
+ + {#if expandedSections.job_retention} +
+

+ Total completed jobs: {formatNumber(data.job_retention.total_completed_jobs)} +

+

+ Oldest job: {formatDate(data.job_retention.oldest_completed_at)} +

+

+ Retention period: + {data.job_retention.retention_period_secs + ? formatNumber(data.job_retention.retention_period_secs) + 's' + : 'Not configured'} + +

+

+ {data.job_retention.message} +

+
+ {/if} +
+ + +
+ + {#if expandedSections.large_results} +
+ {#if data.large_results.top_large_results.length === 0} +

No job results larger than 1 KB found in the scanned jobs.

+ {:else} +
+ + + + + + + + + + + + {#each data.large_results.top_large_results as r} + + + + + + + + {/each} + +
Job IDWorkspaceScriptResult SizeCompleted
{r.id.substring(0, 8)}...{r.workspace_id}{r.runnable_path ?? '-'}{formatBytes(r.result_size_bytes)}{formatDate(r.completed_at)}
+
+ {/if} +
+ {/if} +
+ + +
+ + {#if expandedSections.connection_pool} +
+

+ Total connections: {data.connection_pool.pg_total_connections} / Max: + {data.connection_pool.pg_max_connections} +

+

+ Active: {data.connection_pool.pg_active_connections} + / Idle: {data.connection_pool.pg_idle_connections} +

+

+ {data.connection_pool.message} +

+
+ {/if} +
+ + +
+ + {#if expandedSections.table_maintenance} +
+
+ + + + + + + + + + + + + + {#each data.table_maintenance as t} + + + + + + + + + + {/each} + +
TableLive TuplesDead TuplesDead %Last VacuumLast AnalyzeStatus
{t.table_name}{formatNumber(t.live_tuples)}{formatNumber(t.dead_tuples)}{(t.dead_ratio * 100).toFixed(1)}%{formatDate(t.last_autovacuum)}{formatDate(t.last_autoanalyze)} + + {t.status} + +
+
+
+ {/if} +
+ + +
+ + {#if expandedSections.slow_queries} +
+ {#if data.slow_queries == null} +

Slow query data not available.

+ {:else if data.slow_queries.message} +

{data.slow_queries.message}

+ {:else if data.slow_queries.queries.length === 0} +

No slow queries found.

+ {:else} +
+ + + + + + + + + + + {#each data.slow_queries.queries as q} + + + + + + + {/each} + +
QueryCallsTotal TimeMean Time
{q.query}{formatNumber(q.calls)}{formatMs(q.total_exec_time_ms)}{formatMs(q.mean_exec_time_ms)}
+
+ {/if} +
+ {/if} +
+ + +
+ + {#if expandedSections.datatables} +
+ {#if data.datatables.length === 0} +

No instance-stored datatables found.

+ {:else} +
+ + + + + + + + + + + + {#each data.datatables as dt} + + + + + + + + {/each} + +
WorkspaceNameTableSizeEst. Rows
{dt.workspace_id}{dt.name}{dt.table_name}{dt.size_pretty}{formatNumber(Math.round(dt.estimated_rows))}
+
+ {/if} +
+ {/if} +
+ {:else if !loading} +

+ Click "Run Diagnostics" to analyze your database health. The queries are read-only and + lightweight. +

+ {/if} +
diff --git a/frontend/src/lib/components/mcp/McpScopeSelector.svelte b/frontend/src/lib/components/mcp/McpScopeSelector.svelte index 9fe525dd5c..3d019ff539 100644 --- a/frontend/src/lib/components/mcp/McpScopeSelector.svelte +++ b/frontend/src/lib/components/mcp/McpScopeSelector.svelte @@ -5,9 +5,8 @@ import Popover from '$lib/components/Popover.svelte' import MultiSelect from '$lib/components/select/MultiSelect.svelte' import { safeSelectItems } from '$lib/components/select/utils.svelte' - import FolderPicker from '$lib/components/FolderPicker.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' - import { FlowService, IntegrationService, ScriptService } from '$lib/gen' + import { FlowService, FolderService, IntegrationService, ScriptService } from '$lib/gen' import { mcpEndpointTools } from '$lib/mcpEndpointTools' import InfoIcon from 'lucide-svelte/icons/info' import { SvelteMap } from 'svelte/reactivity' @@ -20,7 +19,10 @@ let { workspaceId, scope = $bindable() }: Props = $props() let selectedMode = $state<'favorites' | 'all' | 'folder' | 'custom'>('favorites') - let selectedFolder = $state('') + let selectedFolders = $state([]) + let allFolders = $state([]) + let loadingFolders = $state(false) + let folderNamesCache = new Map() let selectedScripts = $state([]) let selectedFlows = $state([]) let selectedEndpoints = $state([]) @@ -70,8 +72,10 @@ scopeParts.push(`mcp:endpoints:${selectedEndpoints.join(',')}`) } } else if (selectedMode === 'folder') { - const folderPath = `f/${selectedFolder}/*` - scopeParts = [`mcp:scripts:${folderPath}`, `mcp:flows:${folderPath}`, `mcp:endpoints:*`] + const folderPaths = selectedFolders.map((f) => `f/${f}/*`).join(',') + if (selectedFolders.length > 0) { + scopeParts = [`mcp:scripts:${folderPaths}`, `mcp:flows:${folderPaths}`, `mcp:endpoints:*`] + } } else { scopeParts = [`mcp:${selectedMode}`] } @@ -91,13 +95,35 @@ } }) - // Clear folder when not in folder mode + // Clear folders when not in folder mode, load folder names when entering folder mode $effect(() => { - if (selectedMode !== 'folder') { - selectedFolder = '' + if (selectedMode === 'folder' && workspaceId) { + loadFolderNames(workspaceId) + } else { + selectedFolders = [] } }) + async function loadFolderNames(workspace: string) { + if (folderNamesCache.has(workspace)) { + allFolders = folderNamesCache.get(workspace)! + return + } + try { + loadingFolders = true + const excludedFolders = ['app_groups', 'app_custom', 'app_themes'] + const names = ( + await FolderService.listFolderNames({ workspace }) + ).filter((x) => !excludedFolders.includes(x)) + folderNamesCache.set(workspace, names) + allFolders = names + } catch { + allFolders = [] + } finally { + loadingFolders = false + } + } + // Load hub apps on mount async function getAllApps() { if (allApps.length > 0) return @@ -192,11 +218,42 @@ // Load runnables based on mode $effect(() => { if (workspaceId) { - const folderParam = selectedFolder.length > 0 ? selectedFolder : undefined - getScriptsAndFlows(selectedMode === 'favorites', workspaceId, folderParam) + if (selectedMode === 'folder') { + if (selectedFolders.length > 0) { + loadRunnablesForFolders(workspaceId, selectedFolders) + } else { + includedRunnables = [] + } + } else { + getScriptsAndFlows(selectedMode === 'favorites', workspaceId, undefined) + } } }) + async function getCachedRunnables(workspace: string, folder: string): Promise { + const cacheKey = `${workspace}-false-${folder}` + if (runnablesCache.has(cacheKey)) { + return runnablesCache.get(cacheKey) || [] + } + const [scripts, flows] = await Promise.all([ + getScripts(false, workspace, folder), + getFlows(false, workspace, folder) + ]) + const combined = [...scripts, ...flows] + runnablesCache.set(cacheKey, combined) + return combined + } + + async function loadRunnablesForFolders(workspace: string, folders: string[]) { + try { + loadingRunnables = true + const results = await Promise.all(folders.map((f) => getCachedRunnables(workspace, f))) + includedRunnables = [...new Set(results.flat())] + } finally { + loadingRunnables = false + } + } + // Load all scripts/flows for custom mode $effect(() => { if (selectedMode === 'custom' && workspaceId) { @@ -209,7 +266,7 @@ ? 'Create your first scripts or flows to make them available via MCP.' : selectedMode === 'favorites' ? `You do not have any favorite scripts or flows. You can favorite some scripts and flows to include them, or change the scope to "All scripts/flows" to include all your scripts and flows.` - : `You do not have any scripts or flows in the selected folder.` + : `You do not have any scripts or flows in the selected folder(s).` ) function selectAllScripts() { @@ -252,8 +309,8 @@ - Select Folder - + Select Folders + {#if loadingFolders} +
Loading folders...
+ {:else} + + {/if}
{/if} @@ -389,7 +454,7 @@
{/if} - {:else if selectedMode !== 'folder' || selectedFolder.length > 0} + {:else if selectedMode !== 'folder' || selectedFolders.length > 0} {#if loadingRunnables}
fork(e.detail)} on:delete {id} diff --git a/frontend/src/lib/components/raw_apps/rawAppPolicy.ts b/frontend/src/lib/components/raw_apps/rawAppPolicy.ts index b90f1d3985..ec3e089745 100644 --- a/frontend/src/lib/components/raw_apps/rawAppPolicy.ts +++ b/frontend/src/lib/components/raw_apps/rawAppPolicy.ts @@ -1,18 +1,24 @@ import type { Policy, ScriptLang } from '$lib/gen' import { collectStaticFields, hash, type TriggerableV2 } from '../apps/editor/commonAppUtils' -import { isRunnableByName, isRunnableByPath, type InlineScript, type RunnableWithFields } from '../apps/inputType' +import { + isRunnableByName, + isRunnableByPath, + type InlineScript, + type RunnableWithFields +} from '../apps/inputType' export async function updateRawAppPolicy( runnables: Record, currentPolicy: Policy | undefined ): Promise { - const triggerables_v2 = Object.fromEntries( - (await Promise.all( + const entries = ( + await Promise.all( Object.entries(runnables).map(async ([id, runnable]) => { return await processRunnable(id, runnable, runnable?.fields ?? {}) }) - )) as [string, TriggerableV2][] - ) + ) + ).filter((entry): entry is [string, TriggerableV2] => entry != null) + const triggerables_v2 = Object.fromEntries(entries) return { ...currentPolicy, triggerables_v2 diff --git a/frontend/src/lib/components/raw_apps/utils.test.ts b/frontend/src/lib/components/raw_apps/utils.test.ts new file mode 100644 index 0000000000..e6b1fe5e36 --- /dev/null +++ b/frontend/src/lib/components/raw_apps/utils.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest' + +import { genWmillTs, type Runnable } from './utils' + +const flowSchema = { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + required: ['string_input'], + properties: { + string_input: { + type: 'string', + default: '' + }, + count: { + type: 'integer' + } + } +} as const + +describe('genWmillTs', () => { + it('generates the correct flow args type for path runnables', () => { + const runnables: Record = { + myflow: { + type: 'path', + runType: 'flow', + path: 'u/dev/my_flow', + name: 'My flow', + schema: flowSchema, + fields: { + count: { + type: 'static', + value: 1, + fieldType: 'number' + } + } + } + } + + const dts = genWmillTs(runnables) + + expect(dts).toContain( + 'myflow: (args: { string_input: string }) => Promise;' + ) + }) +}) diff --git a/frontend/src/lib/components/raw_apps/utils.ts b/frontend/src/lib/components/raw_apps/utils.ts index 06264d9c5f..928a7b8d81 100644 --- a/frontend/src/lib/components/raw_apps/utils.ts +++ b/frontend/src/lib/components/raw_apps/utils.ts @@ -108,7 +108,7 @@ function hiddenRunnableToTsType(runnable: Runnable) { } } else if (isRunnableByPath(runnable)) { if (runnable?.schema) { - return schemaToTsType(removeStaticFields(runnable.schema, runnable?.fields ?? {})) + return schemaToTsType(removeStaticFields(runnable.schema as Schema, runnable?.fields ?? {})) } else { return '{}' } diff --git a/frontend/src/lib/components/recording/FlowRecordingReplay.svelte b/frontend/src/lib/components/recording/FlowRecordingReplay.svelte index c5d94a7b95..afd2cee94c 100644 --- a/frontend/src/lib/components/recording/FlowRecordingReplay.svelte +++ b/frontend/src/lib/components/recording/FlowRecordingReplay.svelte @@ -2,7 +2,8 @@ import type { Job } from '$lib/gen' import { workspaceStore } from '$lib/stores' import FlowStatusViewer from '$lib/components/FlowStatusViewer.svelte' - import FlowViewer from '$lib/components/FlowViewer.svelte' + import FlowViewer, { type TabValue } from '$lib/components/FlowViewer.svelte' + import FlowGraphViewer from '$lib/components/FlowGraphViewer.svelte' import FlowProgressBar from '$lib/components/flows/FlowProgressBar.svelte' import FlowExecutionStatus from '$lib/components/runs/FlowExecutionStatus.svelte' import { setActiveReplay } from './flowRecording.svelte' @@ -13,21 +14,37 @@ import { InfoIcon, LogOut, Play, Square } from 'lucide-svelte' import { onDestroy } from 'svelte' - interface Props { - recording: FlowRecording - } - - let { recording }: Props = $props() - type ReplayState = 'loaded' | 'playing' - let replayState: ReplayState = $state('loaded') + interface Props { + recording: FlowRecording + selectedTab?: TabValue + replayState?: ReplayState + hideControls?: boolean + hideTabs?: boolean + } + + let { + recording, + selectedTab = $bindable(), + replayState = $bindable(), + hideControls = false, + hideTabs = false + }: Props = $props() + + if (selectedTab === undefined) { + selectedTab = 'ui' + } + if (replayState === undefined) { + replayState = 'loaded' + } + let rootJobId: string | undefined = $state(undefined) let rootInitialJob: Job | undefined = $state(undefined) let job: Job | undefined = $state(undefined) let done = $derived((job as any)?.type === 'CompletedJob') - function stop() { + export function stop() { setActiveReplay(undefined) job = undefined initRecording() @@ -36,10 +53,7 @@ function findRootJobId(data: FlowRecording): string | undefined { for (const [id, recorded] of Object.entries(data.jobs)) { const j = recorded.initial_job - if ( - (j.job_kind === 'flow' || j.job_kind === 'flowpreview') && - !j.parent_job - ) { + if ((j.job_kind === 'flow' || j.job_kind === 'flowpreview') && !j.parent_job) { return id } } @@ -81,17 +95,19 @@ for (const mod of fs.modules) { const durations = mod.flow_jobs_duration if (durations?.started_at) { - durations.started_at = durations.started_at.map( - (d: string) => offsetDate(d) ?? d - ) + durations.started_at = durations.started_at.map((d: string) => offsetDate(d) ?? d) } } } for (const recorded of Object.values(data.jobs)) { offsetJobTimestamps(recorded.initial_job) + if (recorded.initial_job?.flow_status) offsetFlowStatus(recorded.initial_job.flow_status) for (const event of recorded.events) { - if (event.data?.job) offsetJobTimestamps(event.data.job) + if (event.data?.job) { + offsetJobTimestamps(event.data.job) + if (event.data.job.flow_status) offsetFlowStatus(event.data.job.flow_status) + } if (event.data?.flow_status) offsetFlowStatus(event.data.flow_status) } } @@ -141,22 +157,27 @@ // Push the root's completed event to fire after all sub-job events let completedIdx = -1 for (let i = rootEvents.length - 1; i >= 0; i--) { - if (rootEvents[i].data.completed) { completedIdx = i; break } + if (rootEvents[i].data.completed) { + completedIdx = i + break + } } if (completedIdx >= 0 && rootEvents[completedIdx].t < maxSubJobT) { rootEvents[completedIdx].t = maxSubJobT + 50 } } - function startReplay() { + export function startReplay() { + if (!rootJobId) return // JSON round-trip to unwrap reactive proxies and strip non-cloneable properties const snapshot = JSON.parse(JSON.stringify(recording)) as FlowRecording - fixEventOrdering(snapshot, rootJobId!) - rebaseTimestamps(snapshot, rootJobId!) + fixEventOrdering(snapshot, rootJobId) + rebaseTimestamps(snapshot, rootJobId) setActiveReplay(snapshot) - rootInitialJob = buildInitialJob(snapshot, rootJobId!) + rootInitialJob = buildInitialJob(snapshot, rootJobId) job = undefined replayState = 'playing' + selectedTab = 'ui' } onDestroy(() => { @@ -173,52 +194,81 @@

-{:else if replayState === 'loaded'} +{:else}
-
-
-

{recording.flow_path}

- - - {#snippet text()} - - Recorded {new Date(recording.recorded_at).toLocaleString()} — - {(recording.total_duration_ms / 1000).toFixed(1)}s - - {/snippet} - + {#if !hideControls} +
+
+

+ {replayState === 'playing' ? 'Replaying: ' : ''}{recording.flow_path} +

+ + + {#snippet text()} + + Recorded {new Date(recording.recorded_at).toLocaleString()} — + {(recording.total_duration_ms / 1000).toFixed(1)}s + + {/snippet} + +
+ {#if replayState === 'loaded'} + + {:else} + + {/if}
- -
- -
-{:else if replayState === 'playing' && rootJobId} -
-
-

Replaying: {recording.flow_path}

- -
- - {#if job} - {/if} - + + + {#snippet graphContent()} + {#if replayState === 'playing' && rootJobId} +
+ + {#if job} + + {/if} + +
+ {:else} +
+

Click on a step to see its details

+ +
+ {/if} + {/snippet} +
{/if} diff --git a/frontend/src/lib/components/recording/ScriptRecordingReplay.svelte b/frontend/src/lib/components/recording/ScriptRecordingReplay.svelte index 8c3d28a71a..0327044b3b 100644 --- a/frontend/src/lib/components/recording/ScriptRecordingReplay.svelte +++ b/frontend/src/lib/components/recording/ScriptRecordingReplay.svelte @@ -12,6 +12,7 @@ import { json as jsonLang } from 'svelte-highlight/languages' import HighlightTheme from '$lib/components/HighlightTheme.svelte' import JobArgs from '$lib/components/JobArgs.svelte' + import SchemaForm from '$lib/components/SchemaForm.svelte' import DisplayResult from '$lib/components/DisplayResult.svelte' import LogViewer from '$lib/components/LogViewer.svelte' import { ClipboardCopy, InfoIcon, LogOut, Play, Square } from 'lucide-svelte' @@ -19,15 +20,26 @@ import { onDestroy, tick } from 'svelte' import JobLoader from '$lib/components/JobLoader.svelte' - interface Props { - recording: ScriptRecording - } - - let { recording }: Props = $props() + export type ScriptTabValue = 'parameters' | 'code' | 'args' | 'schema' | 'result' type ReplayState = 'loaded' | 'playing' - let replayState: ReplayState = $state('loaded') + interface Props { + recording: ScriptRecording + selectedTab?: ScriptTabValue + replayState?: ReplayState + hideControls?: boolean + hideTabs?: boolean + } + + let { + recording, + selectedTab = $bindable(), + replayState = $bindable(), + hideControls = false, + hideTabs = false + }: Props = $props() + let jobId: string | undefined = $state(undefined) let job: Job | undefined = $state(undefined) let jobLoader: JobLoader | undefined = $state(undefined) @@ -35,7 +47,18 @@ let scriptRecordingStore = createScriptRecording() - function stop() { + let schema = $derived(recording.schema) + + if (selectedTab === undefined) { + if (schema && recording.args) selectedTab = 'parameters' + else if (recording.args && Object.keys(recording.args).length > 0) selectedTab = 'args' + else selectedTab = 'code' + } + if (replayState === undefined) { + replayState = 'loaded' + } + + export function stop() { setActiveReplay(undefined) job = undefined replayState = 'loaded' @@ -85,13 +108,14 @@ initRecording() - async function startReplay() { + export async function startReplay() { const snapshot = JSON.parse(JSON.stringify(recording)) as ScriptRecording rebaseTimestamps(snapshot) const replayData = scriptRecordingStore.toReplayData(snapshot) setActiveReplay(replayData) job = undefined replayState = 'playing' + selectedTab = 'result' await tick() if (jobLoader && jobId) { jobLoader.watchJob(jobId) @@ -101,8 +125,6 @@ onDestroy(() => { setActiveReplay(undefined) }) - - let schema = $derived(recording.schema) @@ -115,48 +137,141 @@

-{:else if replayState === 'loaded'} -
-
-
-

{recording.script_path || 'Untitled script'}

- {recording.language} - - - {#snippet text()} - - Recorded {new Date(recording.recorded_at).toLocaleString()} — - {(recording.total_duration_ms / 1000).toFixed(1)}s - - {/snippet} - +{:else} +
+ {#if !hideControls} +
+
+

+ {replayState === 'playing' ? 'Replaying: ' : ''}{recording.script_path || + 'Untitled script'} +

+ + {recording.language} + + + + {#snippet text()} + + Recorded {new Date(recording.recorded_at).toLocaleString()} — + {(recording.total_duration_ms / 1000).toFixed(1)}s + + {/snippet} + +
+ {#if replayState === 'loaded'} + + {:else} + + {/if}
- -
- - {#if recording.args && Object.keys(recording.args).length > 0} - {/if} - - + {#if replayState === 'playing'} + + {/if} + + + {#if replayState === 'playing'} + + {/if} + {#if schema && recording.args} + + {/if} + {#if recording.args && Object.keys(recording.args).length > 0} + + {/if} + {#if !schema || !recording.args} + + {/if} {#if schema} {/if} {#snippet content()} + + {#if replayState === 'playing' && jobId} +
+
+

Result

+
+ {#if job !== undefined && job.type === 'CompletedJob' && job.result !== undefined} + + {:else if done} +
+ No output available +
+ {:else} +
+ Waiting for result... +
+ {/if} +
+
+
+

Logs

+
+ +
+
+
+ {/if} + + + {#if schema && recording.args} +
+
+ +
+
+ +
+
+ {/if} + + + {#if recording.args && Object.keys(recording.args).length > 0} +
+ +
+ {/if} + -
+
@@ -180,46 +295,4 @@ {/snippet}
-{:else if replayState === 'playing' && jobId} -
-
-

Replaying: {recording.script_path || 'Untitled script'}

- -
- - - {#if done && job} -
-

Result

-
- {#if job.type === 'CompletedJob' && job.result !== undefined} - - {:else} -
No result available
- {/if} -
-
- {/if} - -
- -
-
{/if} diff --git a/frontend/src/lib/components/runs/JobDetailFieldConfig.ts b/frontend/src/lib/components/runs/JobDetailFieldConfig.ts index 23a7df12cd..50edf9e7cf 100644 --- a/frontend/src/lib/components/runs/JobDetailFieldConfig.ts +++ b/frontend/src/lib/components/runs/JobDetailFieldConfig.ts @@ -1,6 +1,7 @@ import type { Job } from '$lib/gen' import { triggerIconMap } from '$lib/components/triggers/utils' import { formatMemory } from '$lib/utils' +import { flowPathToHref } from '$lib/scripts' import { Calendar, Bot } from 'lucide-svelte' import BarsStaggered from '$lib/components/icons/BarsStaggered.svelte' @@ -240,11 +241,12 @@ export const fieldConfigs: Record = { field: 'script_path', label: 'Path', getValue: (job) => job.script_path || null, - getHref: (job, workspaceId) => { + getHref: (job, _workspaceId) => { if (!job.script_path) return null - const stem = job.job_kind === 'script' ? 'scripts' : 'flows' const isScript = job.job_kind === 'script' - return `/${stem}/get/${isScript ? job.script_hash : job.script_path}` + return isScript + ? `/scripts/get/${job.script_hash}` + : flowPathToHref(job.script_path) } }, diff --git a/frontend/src/lib/components/runs/JobDetailHeader.svelte b/frontend/src/lib/components/runs/JobDetailHeader.svelte index 7ca973dd00..b0941ddcad 100644 --- a/frontend/src/lib/components/runs/JobDetailHeader.svelte +++ b/frontend/src/lib/components/runs/JobDetailHeader.svelte @@ -14,6 +14,7 @@ import Button from '$lib/components/common/button/Button.svelte' import DropdownV2 from '$lib/components/DropdownV2.svelte' import { getRelevantFields, getTriggerInfo, type FieldConfig } from './JobDetailFieldConfig' + import { flowPathToHref } from '$lib/scripts' import { slide } from 'svelte/transition' import { twMerge } from 'tailwind-merge' @@ -385,10 +386,12 @@
- {#if job.script_path && (job.job_kind === 'script' || job.job_kind === 'flow' || job.job_kind === 'singlestepflow')} + {#if job.script_path && (job.job_kind === 'script' || job.job_kind === 'flow' || job.job_kind === 'singlestepflow' || job.job_kind === 'flowpreview')} {@const stem = job.job_kind === 'script' ? 'scripts' : 'flows'} {@const isScript = job.job_kind === 'script'} - {@const viewHref = `${base}/${stem}/get/${isScript ? job?.script_hash : job?.script_path}`} + {@const viewHref = isScript + ? `${base}/${stem}/get/${job?.script_hash}` + : flowPathToHref(job?.script_path ?? '')} import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' import PreprocessedArgsDisplay from '$lib/components/runs/PreprocessedArgsDisplay.svelte' - import { truncateHash } from '$lib/utils' + import { getJobKindDisplayLabel, truncateHash } from '$lib/utils' import { base } from '$lib/base' import { truncateRev } from '$lib/utils' import { workspaceStore } from '$lib/stores' @@ -46,7 +46,7 @@ {/if} {#if job && 'job_kind' in job}
- Job kind: {job.job_kind} + Job kind: {getJobKindDisplayLabel(job.job_kind, job.script_path)}
{/if} {#if job && job.flow_status && job.job_kind === 'script'} diff --git a/frontend/src/lib/components/runs/RunRow.svelte b/frontend/src/lib/components/runs/RunRow.svelte index 3a14c71111..d4b1ac8403 100644 --- a/frontend/src/lib/components/runs/RunRow.svelte +++ b/frontend/src/lib/components/runs/RunRow.svelte @@ -9,6 +9,7 @@ isScriptPreview, msToReadableTime, isFlowPreview, + getJobKindDisplayLabel, getJobKindIcon } from '$lib/utils' import { Button } from '../common' @@ -155,12 +156,12 @@ {/if}
- {#snippet text()} - - {#if job && job.job_kind} - {job.job_kind} - {/if} - {#if job && job.is_flow_step && job.parent_job} + {#snippet text()} + + {#if job && job.job_kind} + {getJobKindDisplayLabel(job.job_kind, job.script_path)} + {/if} + {#if job && job.is_flow_step && job.parent_job}
Step of flow {truncateRev(job.parent_job, 10)} diff --git a/frontend/src/lib/components/schema/EditableSchemaWrapper.svelte b/frontend/src/lib/components/schema/EditableSchemaWrapper.svelte index 27f19928f1..65a865f1d9 100644 --- a/frontend/src/lib/components/schema/EditableSchemaWrapper.svelte +++ b/frontend/src/lib/components/schema/EditableSchemaWrapper.svelte @@ -20,6 +20,7 @@ fullHeight = true, formatExtension = $bindable(undefined), isFileset = $bindable(undefined), + showSensitiveToggle = false, customUi }: EditableSchemaWrapperProps = $props() @@ -113,6 +114,7 @@ bind:this={editableSchemaForm} bind:schema isFlowInput + {showSensitiveToggle} on:delete={(e) => { addPropertyComponent?.handleDeleteArgument([e.detail]) }} @@ -162,9 +164,9 @@ {:else if formatExtension && formatExtension !== ''} - The .{formatExtension} extension will be used to - infer the format when displaying the content and this is also how the resource will appear - when pulling via the CLI. + The .{formatExtension} extension will be used to infer + the format when displaying the content and this is also how the resource will appear when pulling + via the CLI.
{/if} @@ -175,10 +177,7 @@ path and contains text content. In the CLI, filesets are stored as directories. {/if} - switchResourceMode(mode)} -> + switchResourceMode(mode)}> {#snippet children({ item })} diff --git a/frontend/src/lib/components/schema/PropertyEditor.svelte b/frontend/src/lib/components/schema/PropertyEditor.svelte index 1dcf1a817d..3cd0ec5507 100644 --- a/frontend/src/lib/components/schema/PropertyEditor.svelte +++ b/frontend/src/lib/components/schema/PropertyEditor.svelte @@ -7,6 +7,7 @@ import NumberTypeNarrowing from '../NumberTypeNarrowing.svelte' import StringTypeNarrowing from '../StringTypeNarrowing.svelte' import Tooltip from '../Tooltip.svelte' + import Toggle from '../Toggle.svelte' import EditableSchemaForm from '../EditableSchemaForm.svelte' import { deepEqual } from 'fast-equals' @@ -35,6 +36,7 @@ nonEmpty?: boolean | undefined isFlowInput?: boolean isAppInput?: boolean + showSensitiveToggle?: boolean order?: string[] | undefined itemsType?: | { @@ -66,6 +68,7 @@ properties = $bindable(), isFlowInput = false, isAppInput = false, + showSensitiveToggle = false, order = $bindable(), itemsType = $bindable(undefined), typeeditor, @@ -290,5 +293,25 @@ {/if} {@render children?.()} + + {#if type == 'object' && showSensitiveToggle} + { + if (e.detail) { + extra['password'] = true + } else { + extra['password'] = undefined + } + dispatch('change') + }} + /> + {/if}
diff --git a/frontend/src/lib/components/schema/editable_schema_wrapper.ts b/frontend/src/lib/components/schema/editable_schema_wrapper.ts index 1ef4426613..bcc659b27e 100644 --- a/frontend/src/lib/components/schema/editable_schema_wrapper.ts +++ b/frontend/src/lib/components/schema/editable_schema_wrapper.ts @@ -7,6 +7,7 @@ export type EditableSchemaWrapperProps = { fullHeight?: boolean formatExtension?: string | undefined isFileset?: boolean | undefined + showSensitiveToggle?: boolean customUi?: { noAddPopover?: boolean } diff --git a/frontend/src/lib/components/secretArgUtils.ts b/frontend/src/lib/components/secretArgUtils.ts new file mode 100644 index 0000000000..82aee1f53d --- /dev/null +++ b/frontend/src/lib/components/secretArgUtils.ts @@ -0,0 +1,49 @@ +import type { Schema } from '$lib/common' +import { VariableService } from '$lib/gen' +import { get } from 'svelte/store' +import { userStore, workspaceStore } from '$lib/stores' +import { generateRandomString } from '$lib/utils' + +/** + * Process args before job submission: for non-string fields marked as password/sensitive, + * create ephemeral secret variables and replace values with $jsonvar:path references. + * String password fields are already handled by PasswordArgInput (uses $var:). + */ +export async function processSecretArgs( + args: Record, + schema: Schema | undefined +): Promise> { + if (!schema?.properties) return args + + const workspace = get(workspaceStore) + const user = get(userStore) + if (!workspace || !user) return args + + const username = (user.username ?? user.email)?.split('@')[0] + if (!username) return args + const userPrefix = `u/${username}/secret_arg/` + + const result = { ...args } + + for (const [key, prop] of Object.entries(schema.properties)) { + if (!prop.password) continue + if (prop.type !== 'object') continue // only object types; strings handled by PasswordArgInput + if (result[key] == null || result[key] === undefined) continue + if (typeof result[key] === 'string' && result[key].startsWith('$jsonvar:')) continue // already processed + + const path = userPrefix + generateRandomString(12) + await VariableService.createVariable({ + workspace, + requestBody: { + value: JSON.stringify(result[key]), + is_secret: true, + path, + description: 'Ephemeral secret variable', + expires_at: new Date(Date.now() + 1000 * 60 * 60 * 24 * 7).toISOString() + } + }) + result[key] = '$jsonvar:' + path + } + + return result +} diff --git a/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte b/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte index 03b57f6d9a..8b245c7c70 100644 --- a/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte +++ b/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte @@ -14,9 +14,15 @@ import Tooltip from '$lib/components/Tooltip.svelte' import type { CancelablePromise, User, UserUsage } from '$lib/gen' import { UserService, WorkspaceService, GroupService, type WorkspaceInvite } from '$lib/gen' - import { userStore, workspaceStore, superadmin, globalEmailInvite } from '$lib/stores' + import { + userStore, + workspaceStore, + superadmin, + globalEmailInvite, + enterpriseLicense + } from '$lib/stores' import { sendUserToast } from '$lib/toast' - import { Loader2, Mails, Search, Plus, UserMinus, X } from 'lucide-svelte' + import { Loader2, Mails, Search, Plus, UserMinus, X, Bot, LogIn } from 'lucide-svelte' import Select from '$lib/components/select/Select.svelte' import SearchItems from '../SearchItems.svelte' import Cell from '../table/Cell.svelte' @@ -45,6 +51,8 @@ let selectedNewInstanceGroup: string | undefined = $state(undefined) let selectedNewRole: string | undefined = $state('developer') + // Service account creation + // Available groups for dropdowns - filter out already configured groups let availableGroupItems = $derived( instanceGroups @@ -488,12 +496,14 @@ {#snippet children({ item })} {/if} - {truncate(email, 20)} - {truncate(username, 30)} + + {#if user.is_service_account} + + + {email} + + {:else} + {email} + {/if} + + {username} {#if hasNonManualUsers}
@@ -796,14 +825,21 @@ {/if} {#if usage?.[email] != undefined}{usage?.[email]}{:else}{#if usage != undefined}{usage[email] ?? 0}{:else}{/if}
- {#if added_via?.source === 'instance_group'} + {#if user.is_service_account} +
+ + Operator + + Service accounts are always operators. +
+ {:else if added_via?.source === 'instance_group'}
{is_admin ? 'Admin' : operator ? 'Operator' : 'Developer'} @@ -840,6 +876,7 @@ {#snippet children({ item })}
+ {#if user.is_service_account && $userStore?.is_admin} + + {/if} {#snippet removeUserButton(disabled: boolean)}
diff --git a/frontend/src/lib/components/triggers/CaptureWrapper.svelte b/frontend/src/lib/components/triggers/CaptureWrapper.svelte index a6908c6957..0c763987a9 100644 --- a/frontend/src/lib/components/triggers/CaptureWrapper.svelte +++ b/frontend/src/lib/components/triggers/CaptureWrapper.svelte @@ -260,7 +260,7 @@ {hasPreprocessor} {isFlow} {captureLoading} - {triggerDeployed} + groupId={args?.group_id} on:applyArgs on:updateSchema on:addPreprocessor diff --git a/frontend/src/lib/components/triggers/TriggerFilters.svelte b/frontend/src/lib/components/triggers/TriggerFilters.svelte index e2657f6cbd..cb7f198346 100644 --- a/frontend/src/lib/components/triggers/TriggerFilters.svelte +++ b/frontend/src/lib/components/triggers/TriggerFilters.svelte @@ -1,23 +1,45 @@

- Filters will limit the execution of the trigger to only messages that match all criteria.
+ {description}
The JSON filter checks if the value at the key is equal or a superset of the filter value.

+ {#if filters.length > 0} +
+ - -