diff --git a/.claude/settings.json b/.claude/settings.json index fcd49c3140..cf8bfdd284 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -110,7 +110,6 @@ ] }, "enabledPlugins": { - "rust-analyzer-lsp@claude-plugins-official": true, "typescript-lsp@claude-plugins-official": true, "code-review@claude-plugins-official": true } diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml new file mode 100644 index 0000000000..ca9ce2aaac --- /dev/null +++ b/.github/workflows/backend-test-windows.yml @@ -0,0 +1,165 @@ +name: Backend integration tests (Windows) + +on: + workflow_dispatch: + push: + branches: + - "ci-windows-tests" + +env: + CARGO_INCREMENTAL: 0 + SQLX_OFFLINE: true + DISABLE_EMBEDDING: true + +jobs: + cargo_test_windows: + runs-on: blacksmith-16vcpu-windows-2025 + steps: + - uses: actions/checkout@v4 + + - name: Read EE repo commit hash + shell: pwsh + run: | + $ee_repo_ref = Get-Content .\backend\ee-repo-ref.txt + echo "ee_repo_ref=$ee_repo_ref" | Out-File -FilePath $env:GITHUB_ENV -Append + + - name: Checkout windmill-ee-private repository + uses: actions/checkout@v4 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ env.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 0 + + - name: Substitute EE code + shell: bash + run: | + ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + + - name: Setup PostgreSQL + uses: ikalnytskyi/action-setup-postgres@v6 + with: + username: postgres + password: changeme + database: windmill + port: 5432 + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache-workspaces: backend + toolchain: 1.93.0 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: "9.0.x" + + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + + - uses: actions/setup-go@v2 + with: + go-version: 1.21.5 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.10 + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - uses: astral-sh/setup-uv@v6.2.1 + with: + version: "0.9.24" + + - uses: shivammathur/setup-php@v2 + with: + php-version: "8.3" + tools: composer + + - name: Install windmill CLI + shell: bash + run: | + cd cli + bash gen_wm_client.sh + bun install + mkdir -p "$HOME/.local/bin" + printf '#!/bin/sh\nexec bun run "%s/cli/src/main.ts" "$@"\n' "$GITHUB_WORKSPACE" > "$HOME/.local/bin/wmill" + chmod +x "$HOME/.local/bin/wmill" + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install OpenSSL via vcpkg + run: | + vcpkg.exe install openssl-windows:x64-windows + vcpkg.exe install openssl:x64-windows-static + vcpkg.exe integrate install + + - name: Get runtime paths + id: runtime-paths + shell: pwsh + run: | + echo "DENO_PATH=$($(Get-Command deno).Source)" >> $env:GITHUB_OUTPUT + echo "BUN_PATH=$($(Get-Command bun).Source)" >> $env:GITHUB_OUTPUT + echo "NODE_BIN_PATH=$($(Get-Command node).Source)" >> $env:GITHUB_OUTPUT + echo "GO_PATH=$($(Get-Command go).Source)" >> $env:GITHUB_OUTPUT + echo "UV_PATH=$($(Get-Command uv).Source)" >> $env:GITHUB_OUTPUT + echo "PHP_PATH=$($(Get-Command php).Source)" >> $env:GITHUB_OUTPUT + echo "COMPOSER_PATH=$($(Get-Command composer).Source)" >> $env:GITHUB_OUTPUT + echo "POWERSHELL_PATH=$($(Get-Command pwsh).Source)" >> $env:GITHUB_OUTPUT + echo "DOTNET_PATH=$($(Get-Command dotnet).Source)" >> $env:GITHUB_OUTPUT + + - name: Build DuckDB FFI module + working-directory: backend/windmill-duckdb-ffi-internal + timeout-minutes: 30 + run: | + cargo build --release -p windmill_duckdb_ffi_internal + New-Item -ItemType Directory -Path ..\target\debug -Force + Copy-Item target\release\windmill_duckdb_ffi_internal.dll ..\target\debug\ + + - name: Print runtime versions and env + shell: pwsh + run: | + deno --version + bun -v + node --version + go version + python3 --version + php --version + pwsh --version + dotnet --version + echo "TEMP=$env:TEMP" + echo "TMP=$env:TMP" + echo "USERPROFILE=$env:USERPROFILE" + echo "HOME=$env:HOME" + + - name: cargo test + working-directory: backend + timeout-minutes: 60 + env: + DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill + RUST_LOG: "off" + RUST_LOG_STYLE: never + CARGO_NET_GIT_FETCH_WITH_CLI: true + CARGO_BUILD_JOBS: 12 + VCPKGRS_DYNAMIC: 1 + OPENSSL_DIR: ${{ env.VCPKG_INSTALLATION_ROOT }}\installed\x64-windows-static + DENO_PATH: ${{ steps.runtime-paths.outputs.DENO_PATH }} + BUN_PATH: ${{ steps.runtime-paths.outputs.BUN_PATH }} + NODE_BIN_PATH: ${{ steps.runtime-paths.outputs.NODE_BIN_PATH }} + GO_PATH: ${{ steps.runtime-paths.outputs.GO_PATH }} + UV_PATH: ${{ steps.runtime-paths.outputs.UV_PATH }} + PHP_PATH: ${{ steps.runtime-paths.outputs.PHP_PATH }} + COMPOSER_PATH: ${{ steps.runtime-paths.outputs.COMPOSER_PATH }} + POWERSHELL_PATH: ${{ steps.runtime-paths.outputs.POWERSHELL_PATH }} + DOTNET_PATH: ${{ steps.runtime-paths.outputs.DOTNET_PATH }} + WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1 + WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1 + WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1 + run: > + cargo test + --no-fail-fast + --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,csharp,php,quickjs,mcp,run_inline + --all + -- --nocapture --test-threads=10 diff --git a/.webmux.yaml b/.webmux.yaml new file mode 100644 index 0000000000..bd88016043 --- /dev/null +++ b/.webmux.yaml @@ -0,0 +1,65 @@ +# Project display name in the dashboard +name: Windmill + +workspace: + mainBranch: main + worktreeRoot: ../windmill__worktrees + defaultAgent: claude + +startupEnvs: + CARGO_FEATURES: "quickjs" + WM_CLONE_DB: false + USE_RUST_PLUGIN: false + +lifecycleHooks: + postCreate: bash ./scripts/post-create.sh + preRemove: bash ./scripts/pre-remove.sh + +auto_name: + model: gemini-2.5-flash-lite + +# Each service defines a port env var that webmux injects into pane and agent +# process environments when creating a worktree. Ports are auto-assigned: +# base + (slot x step). +services: + - name: backend + portEnv: BACKEND_PORT + portStart: 8000 + portStep: 10 + - name: frontend + portEnv: FRONTEND_PORT + portStart: 3000 + portStep: 10 + +profiles: + default: + runtime: host + yolo: true + envPassthrough: [] + systemPrompt: > + You are running inside a tmux session with other panes running services. + Pane layout (current window): + - Pane 0: this pane (claude agent) + - Pane 1: backend (cargo watch -x run) + - Pane 2: frontend (npm run dev) + To check logs, use: \`tmux capture-pane -t .1 -p -S -50\` (backend) or \`tmux capture-pane -t .2 -p -S -50\` (frontend). + When restarting backend or frontend, make sure to use ${BACKEND_PORT} and ${FRONTEND_PORT}. + Because we are running backend with cargo watch, to verify your changes, just check the logs in the backend pane. No need for cargo check. + panes: + - id: agent + kind: agent + focus: true + - id: backend + kind: command + split: right + command: ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/backend" && PORT=${BACKEND_PORT:-8000} cargo watch -x "run ${CARGO_FEATURES:+--features $CARGO_FEATURES}" + - id: frontend + kind: command + split: bottom + command: ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/frontend" && npm run generate-backend-client && REMOTE=${REMOTE:-http://localhost:${BACKEND_PORT:-8000}} npm run dev -- --port ${FRONTEND_PORT:-3000} --host 0.0.0.0 + +integrations: + github: + linkedRepos: [] + linear: + enabled: true diff --git a/.wmdev.yaml b/.wmdev.yaml deleted file mode 100644 index c028c8f3bf..0000000000 --- a/.wmdev.yaml +++ /dev/null @@ -1,111 +0,0 @@ -name: Windmill - -startupEnvs: - CARGO_FEATURES: "quickjs" - -services: - - name: BE - portEnv: BACKEND_PORT - - name: FE - portEnv: FRONTEND_PORT - -profiles: - default: - name: default - - sandbox: - name: sandbox - image: windmill-sandbox - envPassthrough: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - R2_ENDPOINT - - R2_BUCKET - - R2_PUBLIC_URL - extraMounts: - - hostPath: ~/.ssh - guestPath: /root/.ssh - writable: true - - hostPath: ~/.codex - guestPath: /root/.codex - writable: true - - hostPath: ~/windmill-ee-private - writable: true - - hostPath: ~/windmill-ee-private__worktrees - writable: true - systemPrompt: > - You are running inside a sandboxed container with full permissions. - This worktree is configured with the following ports: - - - Backend: port ${BACKEND_PORT}. - Start with: cd backend && PORT=${BACKEND_PORT} - DATABASE_URL=postgres://postgres:changeme@localhost:5432/windmill - cargo watch -x run - - - Frontend: port ${FRONTEND_PORT}. - Start with: cd frontend && REMOTE=http://localhost:${BACKEND_PORT} - npm run dev -- --port ${FRONTEND_PORT} --host 0.0.0.0 - - --- Screenshots --- - You can take screenshots of the frontend UI and upload them to R2 - for use in PR descriptions. - 1) Take a screenshot: - bunx playwright screenshot --browser chromium - http://localhost:${FRONTEND_PORT}/path/to/page /tmp/screenshot.png - 2) Upload to R2: - aws s3 cp /tmp/screenshot.png - "s3://$(printenv R2_BUCKET)/$(git rev-parse --abbrev-ref HEAD)/screenshot.png" - --endpoint-url "$(printenv R2_ENDPOINT)" - 3) The public URL will be: - $(printenv R2_PUBLIC_URL)//screenshot.png - 4) Include in PR descriptions using markdown image syntax. - - --- Terminal Recordings (asciinema) --- - You can record terminal sessions and upload them for sharing. - asciinema is available on PATH. - - 1) Write a shell script with the commands to demo. Add sleep - delays for readable pacing: - - 0.5s after printing a "$ command" line (lets viewer read it) - - 1.5-2s after command output (lets viewer absorb the result) - - Set GIT_PAGER=cat and PAGER=cat to prevent pager hangs - - 2) Record headlessly: - asciinema rec --headless --overwrite \ - -c "bash /tmp/demo.sh" \ - --window-size 120x50 \ - --title "Description of demo" \ - /tmp/demo.cast - - 3) Upload to asciinema.org: - XDG_DATA_HOME=/tmp/.local/share \ - asciinema upload --server-url https://asciinema.org /tmp/demo.cast - - --- Mermaid Diagrams --- - You can render Mermaid diagrams to SVG using the pre-installed mmdc CLI. - The puppeteer config (no-sandbox + Chromium path) is at /root/.puppeteerrc.json. - - 1) Write a .mmd file with your diagram: - cat > /tmp/diagram.mmd << 'EOF' - graph TD - A[Start] --> B[End] - EOF - - 2) Render to SVG (the -p flag is required): - mmdc -i /tmp/diagram.mmd -o /tmp/diagram.svg -p /root/.puppeteerrc.json - - 3) Upload to R2: - aws s3 cp /tmp/diagram.svg - "s3://$(printenv R2_BUCKET)/$(git rev-parse --abbrev-ref HEAD)/diagram.svg" - --endpoint-url "$(printenv R2_ENDPOINT)" - - 4) The public URL will be: - $(printenv R2_PUBLIC_URL)//diagram.svg - - 5) Include in PR descriptions using markdown image syntax. - - IMPORTANT: Read docs/autonomous-mode.md before starting any work. - -linkedRepos: - - repo: windmill-labs/windmill-ee-private - alias: ee diff --git a/CHANGELOG.md b/CHANGELOG.md index ae23b08efa..feec672d41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,87 @@ # Changelog +## [1.653.0](https://github.com/windmill-labs/windmill/compare/v1.652.0...v1.653.0) (2026-03-10) + + +### Features + +* add indexer time window setting (default 7 days) ([#8290](https://github.com/windmill-labs/windmill/issues/8290)) ([0c4d72c](https://github.com/windmill-labs/windmill/commit/0c4d72cfe38d61cf3f6e9bc31056005f1adb494d)) +* add slack connection fields to workspace settings export/import ([#8287](https://github.com/windmill-labs/windmill/issues/8287)) ([39e77ec](https://github.com/windmill-labs/windmill/commit/39e77ecd002b41630fa8d146ee0f15369656acda)) + + +### Performance Improvements + +* optimize job_stats storage for timestamps and zero-memory jobs ([#8289](https://github.com/windmill-labs/windmill/issues/8289)) ([2d8335d](https://github.com/windmill-labs/windmill/commit/2d8335dc43a7cb182eb5a058119d8b0be067cdfd)) + +## [1.652.0](https://github.com/windmill-labs/windmill/compare/v1.651.1...v1.652.0) (2026-03-09) + + +### Features + +* add secretKeyRef support for package registry and storage credentials ([#8275](https://github.com/windmill-labs/windmill/issues/8275)) ([73d27e9](https://github.com/windmill-labs/windmill/commit/73d27e92dd6ced1602f6328f245fec0fa96860e1)) +* expose OTEL trace context as env vars in job execution ([#8277](https://github.com/windmill-labs/windmill/issues/8277)) ([93f75ad](https://github.com/windmill-labs/windmill/commit/93f75ada5e49036f0d998e3d3d53de4dc2c2e83f)) +* workflow-as-code (WAC) v2 ([#8172](https://github.com/windmill-labs/windmill/issues/8172)) ([a6d4390](https://github.com/windmill-labs/windmill/commit/a6d4390790d21d535df1e9d525bffd577c50d8dc)) + + +### Bug Fixes + +* cli: support deleting linked resources-variables without throwing ([#8248](https://github.com/windmill-labs/windmill/issues/8248)) ([7859bca](https://github.com/windmill-labs/windmill/commit/7859bca6ae80d32a73a46910960afc6812e64115)) +* Database studio fixes ([#8251](https://github.com/windmill-labs/windmill/issues/8251)) ([1d78589](https://github.com/windmill-labs/windmill/commit/1d785899404e8636a206cda9a2914df32a1a5269)) +* **frontend:** unsaved changes dialog when flow already saved ([#8259](https://github.com/windmill-labs/windmill/issues/8259)) ([0330993](https://github.com/windmill-labs/windmill/commit/0330993cb66cdabffcd6e552a0f85a9a3931c62d)) +* gracefully handle uninitialized OTEL tracing proxy port ([#8274](https://github.com/windmill-labs/windmill/issues/8274)) ([8b1fe8f](https://github.com/windmill-labs/windmill/commit/8b1fe8f9de7b0c03655558d0c46cfff71a4b2047)) +* guard iteration picker VirtualList against empty items array ([#8273](https://github.com/windmill-labs/windmill/issues/8273)) ([c97cf60](https://github.com/windmill-labs/windmill/commit/c97cf604ab4a902d89fe873b90dbeb9dabc940eb)), closes [#8272](https://github.com/windmill-labs/windmill/issues/8272) +* mask secrets in OAuth config debug/log output ([#8269](https://github.com/windmill-labs/windmill/issues/8269)) ([e75763d](https://github.com/windmill-labs/windmill/commit/e75763dbe5ffe08e6cde082203596d510c2c3b29)) +* parallel branchall hang on bad stop_after_all_iters_if + results.x.length null ([#8276](https://github.com/windmill-labs/windmill/issues/8276)) ([41e523f](https://github.com/windmill-labs/windmill/commit/41e523f827c4e3d5db525a1f14e24936b0b8af46)) +* redact secrets in set_global_setting log line ([#8270](https://github.com/windmill-labs/windmill/issues/8270)) ([6a0473c](https://github.com/windmill-labs/windmill/commit/6a0473c5783dc0fef2ae82dc5345a5f0596f124d)) +* remove $bindable() fallback values causing props_invalid_value error in oauth settings ([#8265](https://github.com/windmill-labs/windmill/issues/8265)) ([037035e](https://github.com/windmill-labs/windmill/commit/037035e094937827305dad29bd76a495d78bc46f)) +* skip down migrations in potentially_stale checksum comparison ([#8271](https://github.com/windmill-labs/windmill/issues/8271)) ([5ba4029](https://github.com/windmill-labs/windmill/commit/5ba4029d8692b2e6054fca7f45ed4cfded4738ef)) +* sql input horizontal scroll missing after switching flow steps ([#8249](https://github.com/windmill-labs/windmill/issues/8249)) ([ce8ac9c](https://github.com/windmill-labs/windmill/commit/ce8ac9cf52dc17061673b9b72556279c48c26f8e)) +* wmill workspace whoami output ([#8246](https://github.com/windmill-labs/windmill/issues/8246)) ([1ac391a](https://github.com/windmill-labs/windmill/commit/1ac391a795585747fe5911ac41b157556569fedb)) + +## [1.651.1](https://github.com/windmill-labs/windmill/compare/v1.651.0...v1.651.1) (2026-03-05) + + +### Bug Fixes + +* prevent slow loading toast interval from leaking on promise cancellation ([#8240](https://github.com/windmill-labs/windmill/issues/8240)) ([2e582b1](https://github.com/windmill-labs/windmill/commit/2e582b1bc1c299388a3c97cfddff9d0eb92858f2)) +* suppress unused variable warnings on windows builds ([#8241](https://github.com/windmill-labs/windmill/issues/8241)) ([2d58382](https://github.com/windmill-labs/windmill/commit/2d583826dc065c05684d4cd1d1510f0d1f2d9ae9)) + +## [1.651.0](https://github.com/windmill-labs/windmill/compare/v1.650.0...v1.651.0) (2026-03-05) + + +### Features + +* add sandbox annotations, volume mounts, for AI sandbox starting with claude ([#8058](https://github.com/windmill-labs/windmill/issues/8058)) ([5f0ef93](https://github.com/windmill-labs/windmill/commit/5f0ef936d1d5d07d01c8e07e26ec254feebef8fb)) +* hash-based MCP tool names for long paths ([#8133](https://github.com/windmill-labs/windmill/issues/8133)) ([ce041e8](https://github.com/windmill-labs/windmill/commit/ce041e8a5e7ff105df389875d9981f3843d4ce39)) + + +### Bug Fixes + +* **python-client:** add delete_s3_object ([#8216](https://github.com/windmill-labs/windmill/issues/8216)) ([90f4c64](https://github.com/windmill-labs/windmill/commit/90f4c64ee12e1d04ce846ff88d6658f667e194e0)) +* update CLI bun template to match UI template ([#8238](https://github.com/windmill-labs/windmill/issues/8238)) ([a8cbe93](https://github.com/windmill-labs/windmill/commit/a8cbe9396ffc51140dce5582d57f4dc59873304e)) +* write fallback package.json for codebase mode nsjail ([#8239](https://github.com/windmill-labs/windmill/issues/8239)) ([d46913b](https://github.com/windmill-labs/windmill/commit/d46913b74a0ffd41d2323e0355cc81954f09e29d)) + +## [1.650.0](https://github.com/windmill-labs/windmill/compare/v1.649.0...v1.650.0) (2026-03-05) + + +### Features + +* add move, delete, and duplicate to flow node context menu ([#8050](https://github.com/windmill-labs/windmill/issues/8050)) ([c0c9388](https://github.com/windmill-labs/windmill/commit/c0c9388415716ce77d841bd08a46f94e0a529685)) +* add variable and resource types to flow env variables ([#8214](https://github.com/windmill-labs/windmill/issues/8214)) ([164e499](https://github.com/windmill-labs/windmill/commit/164e499c64dc5eb76fcfb0f8cefbad2df244f610)) +* Ducklake typechecker ([#8118](https://github.com/windmill-labs/windmill/issues/8118)) ([53caecf](https://github.com/windmill-labs/windmill/commit/53caecf1da8d76e246178dfb9b86d330f0ec52fd)) +* make WINDMILL_DIR configurable via environment variable ([#8215](https://github.com/windmill-labs/windmill/issues/8215)) ([424ca59](https://github.com/windmill-labs/windmill/commit/424ca59dfe3e730f5388d9cac4ea7e69773614d3)) +* make WM_END_USER_EMAIL display users from different workspaces ([#8208](https://github.com/windmill-labs/windmill/issues/8208)) ([baf2bcf](https://github.com/windmill-labs/windmill/commit/baf2bcf14da0c8c95bdbbf511fcaee48be33948b)) +* persistent Db manager state in URI ([#8134](https://github.com/windmill-labs/windmill/issues/8134)) ([4bf827b](https://github.com/windmill-labs/windmill/commit/4bf827bea4d44aca8c5ff7aa67ad449dbcf00673)) +* replace hub error toasts with warning alerts and add disable hub setting ([#8225](https://github.com/windmill-labs/windmill/issues/8225)) ([63ebae8](https://github.com/windmill-labs/windmill/commit/63ebae8829a6dc47a4e23c8670b514f042c9d4be)) +* token expiration notifications ([#8190](https://github.com/windmill-labs/windmill/issues/8190)) ([e56ccd2](https://github.com/windmill-labs/windmill/commit/e56ccd200be29e6ac8ea2b04a341b1ce78a307f6)) + + +### Bug Fixes + +* handle multipart stream errors gracefully instead of panicking ([#8226](https://github.com/windmill-labs/windmill/issues/8226)) ([19c065b](https://github.com/windmill-labs/windmill/commit/19c065bed5468c484c8e7a50a6b79ab90153cc0e)) +* improve windows compatibility ([077779e](https://github.com/windmill-labs/windmill/commit/077779ec52f7d3e5fcc93951544bf47bd6dc30b6)) +* wrap set_encryption_key in a single database transaction ([#8212](https://github.com/windmill-labs/windmill/issues/8212)) ([62382fd](https://github.com/windmill-labs/windmill/commit/62382fd2869ea0190dd0c0b714f9cbd35ceddd7a)) + ## [1.649.0](https://github.com/windmill-labs/windmill/compare/v1.648.0...v1.649.0) (2026-03-03) diff --git a/CLAUDE.local.md b/CLAUDE.local.md new file mode 100644 index 0000000000..d5fb48da2e --- /dev/null +++ b/CLAUDE.local.md @@ -0,0 +1,19 @@ +# Agent 6 — Dev Environment + +## Running Services + +- **Frontend**: http://localhost:5179 +- **Backend**: http://localhost:8006 (docker) +- **Database**: postgres://postgres:changeme@localhost:5432/windmill + +## Authentication + +Default credentials (auto-configured): +- **Email**: admin@windmill.dev +- **Password**: changeme + +Use these for API calls and Playwright login flows. + +## Playwright MCP + +When testing with Playwright, use **http://localhost:5179** as the base URL. diff --git a/CLAUDE.md b/CLAUDE.md index 4e7afeba8a..0acc541c92 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -26,6 +26,27 @@ Open-source platform for internal tools, workflows, API integrations, background - **Login**: `admin@windmill.dev` / `changeme` - **Instance settings**: navigate to `/#superadmin-settings` +## Banned Patterns + +### `$bindable(default_value)` on optional props + +Using `$bindable(default_value)` on props that can be `undefined` is **banned**. This pattern causes subtle bugs because the default value masks the `undefined` state. + +**Bad:** +```svelte +let { my_prop = $bindable(default_value) }: { my_prop?: string } = $props() +``` + +**Correct alternatives:** + +1. **Use `$derived` with nullish coalescing** — handle the potential `undefined` at the usage site: + ```svelte + let { my_prop = $bindable() }: { my_prop?: string } = $props() + let effective_value = $derived(my_prop ?? default_value) + ``` + +2. **Create a `useMyPropState()` helper** — encapsulate the undefined-handling logic in a reusable function and call it higher in the component tree, so the child component always receives a defined value. + ## Core Principles - Search for existing code to reuse before writing new code diff --git a/Dockerfile b/Dockerfile index 7cca6ab329..0cc19801d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -262,6 +262,12 @@ COPY --from=oven/bun:1.3.10 /usr/local/bin/bun /usr/bin/bun RUN bun install -g windmill-cli \ && ln -s $(bun pm bin -g)/wmill /usr/bin/wmill +# Install Claude Code CLI (used by claude sandbox scripts) +# The installer puts the binary in ~/.local/bin/claude (symlink to ~/.local/share/claude/versions/*) +# Copy it to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root) +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && cp /root/.local/share/claude/versions/* /usr/bin/claude + COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer diff --git a/after-add-note.png b/after-add-note.png new file mode 100644 index 0000000000..a98edbdd89 Binary files /dev/null and b/after-add-note.png differ diff --git a/after-click.png b/after-click.png new file mode 100644 index 0000000000..265f84134f Binary files /dev/null and b/after-click.png differ diff --git a/after-dblclick-empty.png b/after-dblclick-empty.png new file mode 100644 index 0000000000..448d3355e0 Binary files /dev/null and b/after-dblclick-empty.png differ diff --git a/after-exit-note-mode.png b/after-exit-note-mode.png new file mode 100644 index 0000000000..32ef8a8e8a Binary files /dev/null and b/after-exit-note-mode.png differ diff --git a/after-pane-click.png b/after-pane-click.png new file mode 100644 index 0000000000..8f326de92a Binary files /dev/null and b/after-pane-click.png differ diff --git a/after-pane-click2.png b/after-pane-click2.png new file mode 100644 index 0000000000..8f326de92a Binary files /dev/null and b/after-pane-click2.png differ diff --git a/after-selection.png b/after-selection.png new file mode 100644 index 0000000000..8b7f391b7b Binary files /dev/null and b/after-selection.png differ diff --git a/backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json b/backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json new file mode 100644 index 0000000000..d9d9793cd1 --- /dev/null +++ b/backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = now() + interval '60 seconds'\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3 AND lease_until > now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f" +} diff --git a/backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json b/backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json similarity index 51% rename from backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json rename to backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json index 84b10ccba6..409faa032f 100644 --- a/backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json +++ b/backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json @@ -1,16 +1,17 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT token\n FROM token\n WHERE token LIKE concat($1::text, '%')\n LIMIT 1\n ", + "query": "SELECT group_ FROM usr_to_group WHERE usr = $1 AND workspace_id = $2", "describe": { "columns": [ { "ordinal": 0, - "name": "token", + "name": "group_", "type_info": "Varchar" } ], "parameters": { "Left": [ + "Text", "Text" ] }, @@ -18,5 +19,5 @@ false ] }, - "hash": "90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492" + "hash": "015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11" } diff --git a/backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json b/backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json new file mode 100644 index 0000000000..52fb375962 --- /dev/null +++ b/backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT large_file_storage->>'volume_storage' FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e" +} diff --git a/backend/.sqlx/query-0af0e0a1dddeee2021ba060e390e1b60caa3752669636e9fb0817a68121a9451.json b/backend/.sqlx/query-0af0e0a1dddeee2021ba060e390e1b60caa3752669636e9fb0817a68121a9451.json new file mode 100644 index 0000000000..afc74d8e7a --- /dev/null +++ b/backend/.sqlx/query-0af0e0a1dddeee2021ba060e390e1b60caa3752669636e9fb0817a68121a9451.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "0af0e0a1dddeee2021ba060e390e1b60caa3752669636e9fb0817a68121a9451" +} diff --git a/backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json b/backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json new file mode 100644 index 0000000000..6a6b77e650 --- /dev/null +++ b/backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "extra_perms", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4" +} diff --git a/backend/.sqlx/query-0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699.json b/backend/.sqlx/query-0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699.json new file mode 100644 index 0000000000..b8e52cdbe7 --- /dev/null +++ b/backend/.sqlx/query-0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET canceled_by = $2, canceled_reason = $3 WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699" +} diff --git a/backend/.sqlx/query-0d4f28ca0c5697c96711ca7225a9a4013e6ccabb495c371471c9d1287defda8f.json b/backend/.sqlx/query-0d4f28ca0c5697c96711ca7225a9a4013e6ccabb495c371471c9d1287defda8f.json new file mode 100644 index 0000000000..10141b3a78 --- /dev/null +++ b/backend/.sqlx/query-0d4f28ca0c5697c96711ca7225a9a4013e6ccabb495c371471c9d1287defda8f.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT j.id, j.runnable_path, j.args, j.kind::text AS \"kind!\"\n FROM v2_job j\n JOIN v2_job_queue q ON j.id = q.id\n WHERE j.runnable_path = $1\n AND j.kind = 'deploymentcallback'\n ORDER BY j.created_at DESC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "args", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "kind!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + true, + null + ] + }, + "hash": "0d4f28ca0c5697c96711ca7225a9a4013e6ccabb495c371471c9d1287defda8f" +} diff --git a/backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json b/backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json new file mode 100644 index 0000000000..0140324406 --- /dev/null +++ b/backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750" +} diff --git a/backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json b/backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json new file mode 100644 index 0000000000..3f39982319 --- /dev/null +++ b/backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 day' WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6" +} diff --git a/backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json b/backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json new file mode 100644 index 0000000000..9a6ae60a49 --- /dev/null +++ b/backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6" +} diff --git a/backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json b/backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json new file mode 100644 index 0000000000..9514010409 --- /dev/null +++ b/backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e" +} diff --git a/backend/.sqlx/query-1db82007445ff5f644bb607aa28f5747cb50d193475fff5fcfdde37d1bc74636.json b/backend/.sqlx/query-1db82007445ff5f644bb607aa28f5747cb50d193475fff5fcfdde37d1bc74636.json deleted file mode 100644 index aaa1ae945b..0000000000 --- a/backend/.sqlx/query-1db82007445ff5f644bb607aa28f5747cb50d193475fff5fcfdde37d1bc74636.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "1db82007445ff5f644bb607aa28f5747cb50d193475fff5fcfdde37d1bc74636" -} diff --git a/backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json b/backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json new file mode 100644 index 0000000000..fa67a0797d --- /dev/null +++ b/backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at)\n VALUES ($1, $2, $3, $4, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET size_bytes = $3, last_used_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab" +} diff --git a/backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json b/backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json new file mode 100644 index 0000000000..8897e8a7de --- /dev/null +++ b/backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc" +} diff --git a/backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json b/backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json new file mode 100644 index 0000000000..2010b40667 --- /dev/null +++ b/backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98" +} diff --git a/backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json b/backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json new file mode 100644 index 0000000000..2fdb1ae80d --- /dev/null +++ b/backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), updated_by = $5, last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int4", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34" +} diff --git a/backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json b/backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json new file mode 100644 index 0000000000..c73c00c2aa --- /dev/null +++ b/backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json @@ -0,0 +1,76 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n name as \"name!\",\n size_bytes as \"size_bytes!\",\n file_count as \"file_count!\",\n created_at as \"created_at!\",\n created_by as \"created_by!\",\n updated_at,\n updated_by,\n description as \"description!\",\n last_used_at,\n extra_perms as \"extra_perms!\"\n FROM (\n SELECT\n COALESCE(v.name, a.path) as name,\n COALESCE(v.size_bytes, 0) as size_bytes,\n COALESCE(v.file_count, 0) as file_count,\n COALESCE(v.created_at, a.min_created_at) as created_at,\n COALESCE(v.created_by, 'unknown') as created_by,\n v.updated_at,\n v.updated_by,\n COALESCE(v.description, '') as description,\n v.last_used_at,\n COALESCE(v.extra_perms, '{}'::jsonb) as extra_perms\n FROM (\n SELECT path, MIN(created_at) as min_created_at\n FROM asset\n WHERE workspace_id = $1 AND kind = 'volume'\n GROUP BY path\n ) a\n FULL OUTER JOIN volume v ON v.workspace_id = $1 AND v.name = a.path\n WHERE v.workspace_id = $1 OR a.path IS NOT NULL\n ) combined\n ORDER BY name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "size_bytes!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "file_count!", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "created_at!", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "updated_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "updated_by", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "description!", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "last_used_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "extra_perms!", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + true, + true, + null, + true, + null + ] + }, + "hash": "40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468" +} diff --git a/backend/.sqlx/query-56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5.json b/backend/.sqlx/query-56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5.json deleted file mode 100644 index 6b47103c3a..0000000000 --- a/backend/.sqlx/query-56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n COALESCE(workflow_as_code_status, '{}'::jsonb),\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5" -} diff --git a/backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json b/backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json new file mode 100644 index 0000000000..2eda021880 --- /dev/null +++ b/backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM volume WHERE workspace_id = $1 AND name = $2\n AND (lease_until IS NULL OR lease_until < now())\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa" +} diff --git a/backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json b/backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json new file mode 100644 index 0000000000..dd4a011ee1 --- /dev/null +++ b/backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by, extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "extra_perms", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147" +} diff --git a/backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json b/backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json new file mode 100644 index 0000000000..746c306c8f --- /dev/null +++ b/backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM volume WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6" +} diff --git a/backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json b/backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json new file mode 100644 index 0000000000..3df9c6c195 --- /dev/null +++ b/backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int4", + "Text" + ] + }, + "nullable": [] + }, + "hash": "75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca" +} diff --git a/backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json b/backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json new file mode 100644 index 0000000000..0bf454028b --- /dev/null +++ b/backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = now() + interval '60 seconds'\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3 AND lease_until > now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f" +} diff --git a/backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json b/backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json new file mode 100644 index 0000000000..7b85bd9315 --- /dev/null +++ b/backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM volume WHERE workspace_id = $1 AND name = $2 AND lease_until > now() AND leased_by = $3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5" +} diff --git a/backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json b/backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json new file mode 100644 index 0000000000..8cbe7146b6 --- /dev/null +++ b/backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33" +} diff --git a/backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json b/backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json new file mode 100644 index 0000000000..594fcf2960 --- /dev/null +++ b/backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b" +} diff --git a/backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json b/backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json new file mode 100644 index 0000000000..c60239fbce --- /dev/null +++ b/backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "last_used_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4" +} diff --git a/backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json b/backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json new file mode 100644 index 0000000000..b364fcd4ac --- /dev/null +++ b/backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET extra_perms = extra_perms - $1\n WHERE workspace_id = $2 AND name = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d" +} diff --git a/backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json b/backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json new file mode 100644 index 0000000000..728141923a --- /dev/null +++ b/backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2 AND lease_until > now()", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "leased_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a" +} diff --git a/backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json b/backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json new file mode 100644 index 0000000000..26c254eb0b --- /dev/null +++ b/backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a" +} diff --git a/backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json b/backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json new file mode 100644 index 0000000000..ee64e97d12 --- /dev/null +++ b/backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT size_bytes FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7" +} diff --git a/backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json b/backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json new file mode 100644 index 0000000000..fc33fd7373 --- /dev/null +++ b/backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89" +} diff --git a/backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json b/backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json new file mode 100644 index 0000000000..86a99d2eaf --- /dev/null +++ b/backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de" +} diff --git a/backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json b/backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json new file mode 100644 index 0000000000..9051a88a50 --- /dev/null +++ b/backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT extra_perms, created_by FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7" +} diff --git a/backend/.sqlx/query-a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4.json b/backend/.sqlx/query-a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4.json new file mode 100644 index 0000000000..aedbbf424e --- /dev/null +++ b/backend/.sqlx/query-a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4" +} diff --git a/backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json b/backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json new file mode 100644 index 0000000000..1f28a0a5a7 --- /dev/null +++ b/backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, 0, $3)\n ON CONFLICT (workspace_id, name) DO NOTHING\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6" +} diff --git a/backend/.sqlx/query-a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437.json b/backend/.sqlx/query-a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437.json new file mode 100644 index 0000000000..af35d619fa --- /dev/null +++ b/backend/.sqlx/query-a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO token_expiry_notification (token, expiration) VALUES ($1, $2) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437" +} diff --git a/backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json b/backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json new file mode 100644 index 0000000000..7a45e6c402 --- /dev/null +++ b/backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET running = false, started_at = null WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5" +} diff --git a/backend/.sqlx/query-a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4.json b/backend/.sqlx/query-a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4.json new file mode 100644 index 0000000000..00604f4bc9 --- /dev/null +++ b/backend/.sqlx/query-a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token_expiry_notification WHERE expiration <= now()", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4" +} diff --git a/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json b/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json new file mode 100644 index 0000000000..4fa871c594 --- /dev/null +++ b/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e" +} diff --git a/backend/.sqlx/query-a837494a58ab58bfa18c0385350a861076a5fc4f7eefceb1c0de5cf55293f327.json b/backend/.sqlx/query-a837494a58ab58bfa18c0385350a861076a5fc4f7eefceb1c0de5cf55293f327.json new file mode 100644 index 0000000000..dc77dbc402 --- /dev/null +++ b/backend/.sqlx/query-a837494a58ab58bfa18c0385350a861076a5fc4f7eefceb1c0de5cf55293f327.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text", + "Float4" + ] + }, + "nullable": [] + }, + "hash": "a837494a58ab58bfa18c0385350a861076a5fc4f7eefceb1c0de5cf55293f327" +} diff --git a/backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json b/backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json new file mode 100644 index 0000000000..3c7c1ad52a --- /dev/null +++ b/backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935" +} diff --git a/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json b/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json new file mode 100644 index 0000000000..8d09036772 --- /dev/null +++ b/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1 RETURNING suspend", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "suspend", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a" +} diff --git a/backend/.sqlx/query-bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f.json b/backend/.sqlx/query-bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f.json new file mode 100644 index 0000000000..9085383617 --- /dev/null +++ b/backend/.sqlx/query-bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token WHERE expiration <= now()\n RETURNING substring(token for 10) as token_prefix, label, email, workspace_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token_prefix", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "label", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + true, + true, + true + ] + }, + "hash": "bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f" +} diff --git a/backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json b/backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json new file mode 100644 index 0000000000..442d55ed25 --- /dev/null +++ b/backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT size_bytes, last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "last_used_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb" +} diff --git a/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json b/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json new file mode 100644 index 0000000000..efd03ae26e --- /dev/null +++ b/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n workflow_as_code_status,\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3 AND workflow_as_code_status IS NOT NULL\n RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS \"job_ids: serde_json::Value\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_ids: serde_json::Value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340" +} diff --git a/backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json b/backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json new file mode 100644 index 0000000000..0400e6992d --- /dev/null +++ b/backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, name, size_bytes, created_by, last_used_at\n FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "last_used_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true + ] + }, + "hash": "d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c" +} diff --git a/backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json b/backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json new file mode 100644 index 0000000000..16fc965f6a --- /dev/null +++ b/backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "leased_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04" +} diff --git a/backend/.sqlx/query-d44c37882150532383d1058639f4d3af288a346c50f29809dbc55a34088a0abc.json b/backend/.sqlx/query-d44c37882150532383d1058639f4d3af288a346c50f29809dbc55a34088a0abc.json deleted file mode 100644 index c071272ff9..0000000000 --- a/backend/.sqlx/query-d44c37882150532383d1058639f4d3af288a346c50f29809dbc55a34088a0abc.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text", - "Float4" - ] - }, - "nullable": [] - }, - "hash": "d44c37882150532383d1058639f4d3af288a346c50f29809dbc55a34088a0abc" -} diff --git a/backend/.sqlx/query-d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6.json b/backend/.sqlx/query-d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6.json new file mode 100644 index 0000000000..015aa7b05a --- /dev/null +++ b/backend/.sqlx/query-d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token_expiry_notification n\n USING token t\n WHERE n.token = t.token\n AND n.expiration > now()\n AND n.expiration <= now() + interval '7 days'\n RETURNING substring(t.token for 10) as token_prefix, t.label, t.email, t.workspace_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token_prefix", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "label", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + true, + true, + true + ] + }, + "hash": "d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6" +} diff --git a/backend/.sqlx/query-1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82.json b/backend/.sqlx/query-dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353.json similarity index 65% rename from backend/.sqlx/query-1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82.json rename to backend/.sqlx/query-dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353.json index 5b82c5288f..5791090fc1 100644 --- a/backend/.sqlx/query-1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82.json +++ b/backend/.sqlx/query-dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n WITH job_info AS (\n SELECT id, kind::text AS kind, parent_job\n FROM v2_job\n WHERE id = $1\n )\n SELECT\n q.id AS \"id!\",\n s.flow_status,\n q.suspend AS \"suspend!\",\n j.runnable_path AS script_path,\n j.permissioned_as_email AS email,\n (ji.kind IN ('flow', 'flowpreview')) AS \"is_flow_level!\"\n FROM job_info ji\n JOIN v2_job_queue q ON q.id = CASE\n WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id\n ELSE ji.parent_job\n END\n JOIN v2_job j ON j.id = q.id\n JOIN v2_job_status s ON s.id = q.id\n FOR UPDATE OF q\n ", + "query": "\n WITH job_info AS (\n SELECT id, kind::text AS kind, parent_job\n FROM v2_job\n WHERE id = $1\n )\n SELECT\n q.id AS \"id!\",\n s.flow_status,\n q.suspend AS \"suspend!\",\n j.runnable_path AS script_path,\n j.permissioned_as_email AS email,\n (ji.kind IN ('flow', 'flowpreview')) AS \"is_flow_level!\",\n (ji.kind NOT IN ('flow', 'flowpreview') AND q.id = ji.id) AS \"is_wac!\"\n FROM job_info ji\n JOIN v2_job_queue q ON q.id = CASE\n WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id\n ELSE COALESCE(ji.parent_job, ji.id)\n END\n JOIN v2_job j ON j.id = q.id\n LEFT JOIN v2_job_status s ON s.id = q.id\n FOR UPDATE OF q\n ", "describe": { "columns": [ { @@ -32,6 +32,11 @@ "ordinal": 5, "name": "is_flow_level!", "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "is_wac!", + "type_info": "Bool" } ], "parameters": { @@ -45,8 +50,9 @@ false, true, false, + null, null ] }, - "hash": "1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82" + "hash": "dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353" } diff --git a/backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json b/backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json new file mode 100644 index 0000000000..91d89df9cd --- /dev/null +++ b/backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, size_bytes FROM volume WHERE workspace_id = $1 ORDER BY name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "size_bytes", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865" +} diff --git a/backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json b/backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json new file mode 100644 index 0000000000..c4a1d4cd07 --- /dev/null +++ b/backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2::bool), true)\n WHERE workspace_id = $3 AND name = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Bool", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d" +} diff --git a/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json new file mode 100644 index 0000000000..c96961eac4 --- /dev/null +++ b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT token as \"token!\"\n FROM token\n WHERE token LIKE concat($1::text, '%')\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06" +} diff --git a/backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json b/backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json new file mode 100644 index 0000000000..649ee4c387 --- /dev/null +++ b/backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT permissioned_as FROM v2_job WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "permissioned_as", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0" +} diff --git a/backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json b/backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json new file mode 100644 index 0000000000..72acab6120 --- /dev/null +++ b/backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20" +} diff --git a/backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json b/backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json new file mode 100644 index 0000000000..e1ca416938 --- /dev/null +++ b/backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT size_bytes, file_count, leased_by, lease_until\n FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "file_count", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "leased_by", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "lease_until", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + true + ] + }, + "hash": "f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index d26d5b76d7..64a04ce4ec 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1900,7 +1900,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0686c856aa6aac0c4498f936d7d6a02df690f614c03e4d906d1018062b5c5e2c" dependencies = [ "once_cell", - "proc-macro-crate 3.4.0", + "proc-macro-crate", "proc-macro2", "quote", "syn 2.0.117", @@ -2550,6 +2550,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cooked-waker" version = "5.0.0" @@ -7094,7 +7103,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.2", + "socket2 0.6.3", "system-configuration", "tokio", "tower-service", @@ -7965,9 +7974,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.182" +version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "libffi" @@ -8093,9 +8102,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.24" +version = "1.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4735e9cbde5aac84a5ce588f6b23a90b9b0b528f6c5a8db8a4aff300463a0839" +checksum = "d52f4c29e2a68ac30c9087e1b772dc9f44a2b66ed44edf2266cf2be9b03dafc1" dependencies = [ "cc", "libc", @@ -8683,7 +8692,7 @@ dependencies = [ "darling 0.20.11", "heck 0.5.0", "num-bigint", - "proc-macro-crate 3.4.0", + "proc-macro-crate", "proc-macro-error2", "proc-macro2", "quote", @@ -9252,7 +9261,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ - "proc-macro-crate 3.4.0", + "proc-macro-crate", "proc-macro2", "quote", "syn 2.0.117", @@ -10331,16 +10340,6 @@ dependencies = [ "elliptic-curve", ] -[[package]] -name = "proc-macro-crate" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" -dependencies = [ - "once_cell", - "toml_edit 0.19.15", -] - [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -10665,7 +10664,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.35", - "socket2 0.6.2", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -10674,9 +10673,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "aws-lc-rs", "bytes", @@ -10703,16 +10702,16 @@ dependencies = [ "cfg_aliases 0.2.1", "libc", "once_cell", - "socket2 0.6.2", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.44" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -11094,9 +11093,12 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "relative-path" -version = "1.9.3" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" +checksum = "bca40a312222d8ba74837cb474edef44b37f561da5f773981007a10bbaa992b0" +dependencies = [ + "serde", +] [[package]] name = "rend" @@ -11390,9 +11392,9 @@ dependencies = [ [[package]] name = "rquickjs" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16661bff09e9ed8e01094a188b463de45ec0693ade55b92ed54027d7ba7c40c" +checksum = "c50dc6d6c587c339edb4769cf705867497a2baf0eca8b4645fa6ecd22f02c77a" dependencies = [ "rquickjs-core", "rquickjs-macro", @@ -11400,26 +11402,27 @@ dependencies = [ [[package]] name = "rquickjs-core" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8db6379e204ef84c0811e90e7cc3e3e4d7688701db68a00d14a6db6849087b" +checksum = "b8bf7840285c321c3ab20e752a9afb95548c75cd7f4632a0627cea3507e310c1" dependencies = [ "async-lock", + "hashbrown 0.16.0", "relative-path", "rquickjs-sys", ] [[package]] name = "rquickjs-macro" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6041104330c019fcd936026ae05e2446f5e8a2abef329d924f25424b7052a2f3" +checksum = "7106215ff41a5677b104906a13e1a440b880f4b6362b5dc4f3978c267fad2b80" dependencies = [ - "convert_case 0.6.0", + "convert_case 0.10.0", "fnv", "ident_case", "indexmap 2.11.1", - "proc-macro-crate 1.3.1", + "proc-macro-crate", "proc-macro2", "quote", "rquickjs-core", @@ -11428,9 +11431,9 @@ dependencies = [ [[package]] name = "rquickjs-sys" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bc352c6b663604c3c186c000cfcc6c271f4b50bc135a285dd6d4f2a42f9790a" +checksum = "27344601ef27460e82d6a4e1ecb9e7e99f518122095f3c51296da8e9be2b9d83" dependencies = [ "cc", ] @@ -12674,12 +12677,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -14459,7 +14462,7 @@ dependencies = [ "indexmap 2.11.1", "toml_datetime 0.7.0", "toml_parser", - "winnow 0.7.14", + "winnow 0.7.15", ] [[package]] @@ -14468,7 +14471,7 @@ version = "1.0.9+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" dependencies = [ - "winnow 0.7.14", + "winnow 0.7.15", ] [[package]] @@ -15738,7 +15741,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-nats", @@ -15770,6 +15773,7 @@ dependencies = [ "sql-builder", "sqlx", "strum 0.27.2", + "tar", "tempfile", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", @@ -15795,14 +15799,16 @@ dependencies = [ "windmill-queue", "windmill-runtime-nativets", "windmill-test-utils", + "windmill-types", "windmill-worker", + "windmill-worker-volumes", "windows-service", "windows-sys 0.52.0", ] [[package]] name = "windmill-alerting" -version = "1.649.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15815,7 +15821,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "argon2", @@ -15843,6 +15849,7 @@ dependencies = [ "dashmap 6.1.0", "datafusion", "ed25519-dalek", + "eventsource-stream", "flate2", "futures", "git-version", @@ -15949,11 +15956,12 @@ dependencies = [ "windmill-trigger-websocket", "windmill-types", "windmill-worker", + "windmill-worker-volumes", ] [[package]] name = "windmill-api-agent-workers" -version = "1.649.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15976,7 +15984,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.649.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15989,7 +15997,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16015,7 +16023,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.649.0" +version = "1.653.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -16025,7 +16033,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.649.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16042,7 +16050,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.649.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "base64 0.22.1", @@ -16065,7 +16073,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16088,7 +16096,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.649.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16104,7 +16112,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.649.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16124,7 +16132,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.649.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16144,7 +16152,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.649.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16158,7 +16166,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-nats", @@ -16185,7 +16193,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16210,7 +16218,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.649.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "flate2", @@ -16228,7 +16236,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16249,7 +16257,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.649.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16269,7 +16277,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.649.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16299,7 +16307,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16326,7 +16334,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.649.0" +version = "1.653.0" dependencies = [ "lazy_static", "serde", @@ -16338,7 +16346,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.649.0" +version = "1.653.0" dependencies = [ "argon2", "axum 0.7.9", @@ -16361,7 +16369,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.649.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16375,7 +16383,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.649.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16383,6 +16391,7 @@ dependencies = [ "http 1.4.0", "hyper 1.8.1", "lazy_static", + "magic-crypt", "regex", "serde", "serde_json", @@ -16405,7 +16414,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.649.0" +version = "1.653.0" dependencies = [ "chrono", "lazy_static", @@ -16419,7 +16428,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16438,7 +16447,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.649.0" +version = "1.653.0" dependencies = [ "aes-gcm", "anyhow", @@ -16537,7 +16546,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.649.0" +version = "1.653.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -16556,7 +16565,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.649.0" +version = "1.653.0" dependencies = [ "regex", "serde", @@ -16571,7 +16580,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -16595,7 +16604,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "futures", @@ -16612,7 +16621,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.649.0" +version = "1.653.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -16628,7 +16637,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -16649,7 +16658,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -16680,7 +16689,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-oauth2", @@ -16704,7 +16713,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-stream", @@ -16738,7 +16747,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "futures", @@ -16756,7 +16765,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.649.0" +version = "1.653.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -16765,7 +16774,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "lazy_static", @@ -16777,7 +16786,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "serde_json", @@ -16789,7 +16798,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "gosyn", @@ -16801,7 +16810,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "lazy_static", @@ -16813,7 +16822,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "serde_json", @@ -16825,7 +16834,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "nu-parser", @@ -16836,7 +16845,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16847,7 +16856,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16860,7 +16869,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-recursion", @@ -16884,7 +16893,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "lazy_static", @@ -16898,7 +16907,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16915,7 +16924,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "lazy_static", @@ -16930,7 +16939,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "lazy_static", @@ -16947,9 +16956,25 @@ dependencies = [ "windmill-parser-sql", ] +[[package]] +name = "windmill-parser-wac" +version = "1.653.0" +dependencies = [ + "anyhow", + "rustpython-ast", + "rustpython-parser", + "serde", + "serde_json", + "sha2 0.10.9", + "swc_common", + "swc_ecma_ast", + "swc_ecma_parser", + "swc_ecma_visit", +] + [[package]] name = "windmill-parser-yaml" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "serde", @@ -16960,7 +16985,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-recursion", @@ -16997,7 +17022,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "const_format", @@ -17035,7 +17060,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.649.0" +version = "1.653.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -17046,7 +17071,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-recursion", @@ -17075,7 +17100,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -17098,7 +17123,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17131,7 +17156,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17151,7 +17176,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17185,7 +17210,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17220,7 +17245,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17243,7 +17268,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17267,7 +17292,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-nats", @@ -17291,7 +17316,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17326,7 +17351,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17354,7 +17379,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17377,7 +17402,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "bitflags 2.9.4", @@ -17395,7 +17420,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.649.0" +version = "1.653.0" dependencies = [ "anyhow", "async-once-cell", @@ -17495,9 +17520,28 @@ dependencies = [ "windmill-queue", "windmill-runtime-nativets", "windmill-types", + "windmill-worker-volumes", "yaml-rust", ] +[[package]] +name = "windmill-worker-volumes" +version = "1.653.0" +dependencies = [ + "bytes", + "futures", + "lazy_static", + "md-5 0.10.6", + "object_store", + "regex", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "windmill-common", +] + [[package]] name = "windows" version = "0.56.0" @@ -18082,9 +18126,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" dependencies = [ "memchr", ] @@ -18365,18 +18409,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.40" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.40" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" dependencies = [ "proc-macro2", "quote", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 195d8bd30d..36eca6296b 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.649.0" +version = "1.653.0" authors.workspace = true edition.workspace = true @@ -68,15 +68,17 @@ members = [ "./parsers/windmill-parser-bash", "./parsers/windmill-parser-py", "./parsers/windmill-parser-py-imports", + "./parsers/windmill-parser-wac", "./parsers/windmill-sql-datatype-parser-wasm", "./parsers/windmill-parser-yaml", "windmill-macros", "parsers/windmill-parser-nu", + "./windmill-worker-volumes", "./windmill-test-utils", "./windmill-api-integration-tests", ] exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.649.0" +version = "1.653.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -250,10 +252,13 @@ reqwest.workspace = true windmill-queue = { workspace = true, features = ["failpoints"] } windmill-dep-map.workspace = true windmill-test-utils.workspace = true +windmill-worker-volumes.workspace = true +windmill-types.workspace = true axum.workspace = true serde.workspace = true windmill-api-client.workspace = true tempfile.workspace = true +tar.workspace = true windmill-parser-ts.workspace = true rumqttc.workspace = true rdkafka.workspace = true @@ -267,6 +272,7 @@ aws-credential-types.workspace = true windmill-api = { path = "./windmill-api", default-features = false } windmill-queue = { path = "./windmill-queue" } windmill-worker = { path = "./windmill-worker" } +windmill-worker-volumes = { path = "./windmill-worker-volumes" } windmill-dep-map = { path = "./windmill-dep-map" } windmill-types = { path = "./windmill-types" } windmill-common = { path = "./windmill-common", default-features = false } @@ -327,6 +333,7 @@ windmill-parser-bash = { path = "./parsers/windmill-parser-bash" } windmill-parser-sql = { path = "./parsers/windmill-parser-sql" } windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" } windmill-parser-php = { path = "./parsers/windmill-parser-php" } +windmill-parser-wac = { path = "./parsers/windmill-parser-wac" } windmill-jseval = { path = "./windmill-jseval" } windmill-runtime-nativets = { path = "./windmill-runtime-nativets" } windmill-api-client = { path = "./windmill-api-client" } @@ -439,6 +446,7 @@ base64 = "^0.22.1" base32 = "^0" hmac = "0.12.1" sha2 = "0.10.6" +md-5 = "0.10.6" sha1 = "0.10.6" sqlx = { version = "0.8.0", features = [ "macros", @@ -512,7 +520,7 @@ nu-parser = { version = "0.101.0", default-features = false } globset = "0.4.16" croner = "2.2.0" rmcp = { version = "=0.15.0", features = ["client", "transport-streamable-http-client", "transport-streamable-http-client-reqwest"] } -rquickjs = { version = "0.8", features = ["futures", "parallel", "macro"] } +rquickjs = { version = "0.11", features = ["futures", "parallel", "macro"] } process-wrap = { version = "8.2.1", features = ["tokio1"] } systemstat = "0.2.4" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index a71cab586d..0e277038b7 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -6fd5a2ce908235a17975ad4dbdf0051cd89334f3 +09dfb247f6f59c61b7f2431932c4557fb26c22d8 diff --git a/backend/migrations/20260226000000_add_volumes.down.sql b/backend/migrations/20260226000000_add_volumes.down.sql new file mode 100644 index 0000000000..33dc3804be --- /dev/null +++ b/backend/migrations/20260226000000_add_volumes.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS volume; diff --git a/backend/migrations/20260226000000_add_volumes.up.sql b/backend/migrations/20260226000000_add_volumes.up.sql new file mode 100644 index 0000000000..00f40c9768 --- /dev/null +++ b/backend/migrations/20260226000000_add_volumes.up.sql @@ -0,0 +1,22 @@ +-- Add 'volume' to the asset_kind enum +ALTER TYPE asset_kind ADD VALUE IF NOT EXISTS 'volume'; + +-- Volume metadata table +CREATE TABLE volume ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, + size_bytes BIGINT NOT NULL DEFAULT 0, + file_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by VARCHAR(255) NOT NULL, + updated_at TIMESTAMPTZ, + updated_by VARCHAR(255), + description TEXT NOT NULL DEFAULT '', + lease_until TIMESTAMPTZ, + leased_by VARCHAR(255), + last_used_at TIMESTAMPTZ, + extra_perms JSONB NOT NULL DEFAULT '{}', + PRIMARY KEY (workspace_id, name) +); + +CREATE INDEX idx_volume_last_used ON volume(workspace_id, last_used_at); diff --git a/backend/migrations/20260302000000_add_token_expiry_notified.down.sql b/backend/migrations/20260302000000_add_token_expiry_notified.down.sql new file mode 100644 index 0000000000..ab827c5de5 --- /dev/null +++ b/backend/migrations/20260302000000_add_token_expiry_notified.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS token_expiry_notification; diff --git a/backend/migrations/20260302000000_add_token_expiry_notified.up.sql b/backend/migrations/20260302000000_add_token_expiry_notified.up.sql new file mode 100644 index 0000000000..61883f070d --- /dev/null +++ b/backend/migrations/20260302000000_add_token_expiry_notified.up.sql @@ -0,0 +1,8 @@ +-- Tracks pending expiry notifications: row exists = not yet notified. +-- Deleted once the notification is sent. Orphaned rows are harmless (filtered out by the join). +CREATE TABLE token_expiry_notification ( + token VARCHAR(255) PRIMARY KEY, + expiration TIMESTAMPTZ NOT NULL +); + +CREATE INDEX idx_token_expiry_notification_expiration ON token_expiry_notification (expiration); diff --git a/backend/migrations/20260311000000_optimize_job_stats_timestamps.down.sql b/backend/migrations/20260311000000_optimize_job_stats_timestamps.down.sql new file mode 100644 index 0000000000..e526b89b11 --- /dev/null +++ b/backend/migrations/20260311000000_optimize_job_stats_timestamps.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE job_stats DROP COLUMN IF EXISTS timeseries_start; +ALTER TABLE job_stats DROP COLUMN IF EXISTS offsets_cs; diff --git a/backend/migrations/20260311000000_optimize_job_stats_timestamps.up.sql b/backend/migrations/20260311000000_optimize_job_stats_timestamps.up.sql new file mode 100644 index 0000000000..f851c7887e --- /dev/null +++ b/backend/migrations/20260311000000_optimize_job_stats_timestamps.up.sql @@ -0,0 +1,5 @@ +-- Store timeseries timestamps as a start time + integer centisecond offsets +-- instead of full TIMESTAMPTZ[] arrays. Saves ~4 bytes per data point. +-- i32 centiseconds gives ~248 days of range with 10ms precision. +ALTER TABLE job_stats ADD COLUMN IF NOT EXISTS timeseries_start TIMESTAMPTZ; +ALTER TABLE job_stats ADD COLUMN IF NOT EXISTS offsets_cs INTEGER[]; diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index c299bff4af..655a1a4ea9 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -296,11 +296,14 @@ pub fn parse_python_signature( // Check if main function was found if params.is_none() { + let is_wac_v2 = (code.contains("@workflow") || code.contains("workflow(")) + && (code.contains("@task") || code.contains("task(")) + && (code.contains("import wmill") || code.contains("from wmill")); return Ok(MainArgSignature { star_args: false, star_kwargs: false, args: vec![], - no_main_func: Some(true), + no_main_func: Some(!is_wac_v2), has_preprocessor: Some(has_preprocessor), }); } diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index cc86ea595c..56cccbabfe 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -238,7 +238,7 @@ lazy_static::lazy_static! { // used for `unsafe` sql interpolation // -- %%name%% (type) = default - static ref RE_ARG_SQL_INTERPOLATION: Regex = Regex::new(r#"(?m)^--\s*%%([a-z_][a-z0-9_]*)%%\s*([\s\w\/]+)?(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); + static ref RE_ARG_SQL_INTERPOLATION: Regex = Regex::new(r#"(?m)^--\s*%%([a-z_][a-z0-9_]*)%%[ \t]*([\w][\w \t\/]*)?(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); } fn parsed_default(parsed_typ: &Typ, default: String) -> Option { @@ -1547,4 +1547,36 @@ SELECT $1::integer; Ok(()) } + + #[test] + fn test_parse_pgsql_safe_interpolated_args() -> anyhow::Result<()> { + // There was a bug where enum would be "angrycreative"/"bishop"/"test SELECT x" + let code = r#" +-- %%table_name%% angrycreative/bishop/test +SELECT x +"#; + assert_eq!( + parse_pgsql_sig(code)?, + MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![Arg { + otyp: Some("__sanitized_enum__".to_string()), + name: "table_name".to_string(), + typ: Typ::Str(Some(vec![ + "angrycreative".to_string(), + "bishop".to_string(), + "test".to_string() + ])), + default: None, + has_default: false, + oidx: None, + },], + no_main_func: None, + has_preprocessor: None + } + ); + + Ok(()) + } } diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index 04dd345b2f..f0928fe984 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -261,7 +261,9 @@ pub fn parse_deno_signature( for specifier in &named_export.specifiers { if let swc_ecma_ast::ExportSpecifier::Named(spec) = specifier { let export_name = match &spec.exported { - Some(swc_ecma_ast::ModuleExportName::Ident(ident)) => ident.sym.as_ref(), + Some(swc_ecma_ast::ModuleExportName::Ident(ident)) => { + ident.sym.as_ref() + } Some(swc_ecma_ast::ModuleExportName::Str(s)) => s.value.as_ref(), None => match &spec.orig { swc_ecma_ast::ModuleExportName::Ident(ident) => ident.sym.as_ref(), @@ -315,7 +317,11 @@ pub fn parse_deno_signature( let mut c: u16 = 0; - let no_main_func = entrypoint_params.is_none(); + let is_wac_v2 = entrypoint_params.is_none() + && code.contains("workflow(") + && code.contains("task(") + && code.contains("windmill-client"); + let no_main_func = entrypoint_params.is_none() && !is_wac_v2; let mut type_resolver = HashMap::new(); let r = MainArgSignature { star_args: false, @@ -833,7 +839,9 @@ fn tstype_to_typ( false, ), symbol @ _ if symbol.starts_with("DynMultiselect_") => ( - Typ::DynMultiselect(symbol.strip_prefix("DynMultiselect_").unwrap().to_string()), + Typ::DynMultiselect( + symbol.strip_prefix("DynMultiselect_").unwrap().to_string(), + ), false, ), symbol @ _ => { diff --git a/backend/parsers/windmill-parser-wac/Cargo.toml b/backend/parsers/windmill-parser-wac/Cargo.toml new file mode 100644 index 0000000000..d354b1c653 --- /dev/null +++ b/backend/parsers/windmill-parser-wac/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "windmill-parser-wac" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser_wac" +path = "./src/lib.rs" + +[dependencies] +rustpython-parser.workspace = true +rustpython-ast = { version = "0.4.0", features = ["visitor"] } +swc_common.workspace = true +swc_ecma_parser.workspace = true +swc_ecma_ast.workspace = true +swc_ecma_visit.workspace = true +serde.workspace = true +serde_json.workspace = true +anyhow.workspace = true +sha2.workspace = true diff --git a/backend/parsers/windmill-parser-wac/src/dag.rs b/backend/parsers/windmill-parser-wac/src/dag.rs new file mode 100644 index 0000000000..662f5a06b6 --- /dev/null +++ b/backend/parsers/windmill-parser-wac/src/dag.rs @@ -0,0 +1,44 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct WorkflowDag { + pub nodes: Vec, + pub edges: Vec, + pub params: Vec, + pub source_hash: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct Param { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub typ: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct DagNode { + pub id: String, + pub node_type: DagNodeType, + pub label: String, + pub line: usize, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(tag = "type")] +pub enum DagNodeType { + Step { name: String, script: String }, + Branch { condition_source: String }, + ParallelStart, + ParallelEnd, + LoopStart { iter_source: String }, + LoopEnd, + Return, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct DagEdge { + pub from: String, + pub to: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub label: Option, +} diff --git a/backend/parsers/windmill-parser-wac/src/lib.rs b/backend/parsers/windmill-parser-wac/src/lib.rs new file mode 100644 index 0000000000..1496b9d6cf --- /dev/null +++ b/backend/parsers/windmill-parser-wac/src/lib.rs @@ -0,0 +1,32 @@ +pub mod dag; +pub mod python; +pub mod typescript; +pub mod validation; + +use dag::WorkflowDag; +use validation::CompileError; + +#[derive(Debug, serde::Serialize)] +#[serde(tag = "type")] +pub enum ParseResult { + #[serde(rename = "success")] + Success(WorkflowDag), + #[serde(rename = "error")] + Error { errors: Vec }, +} + +pub fn parse_workflow(code: &str, language: &str) -> ParseResult { + let result = match language { + "python" | "python3" | "py" => python::parse_python_workflow(code), + "typescript" | "ts" | "deno" | "bun" => typescript::parse_ts_workflow(code), + _ => Err(vec![CompileError { + message: format!("Unsupported language: {language}"), + line: 0, + }]), + }; + + match result { + Ok(dag) => ParseResult::Success(dag), + Err(errors) => ParseResult::Error { errors }, + } +} diff --git a/backend/parsers/windmill-parser-wac/src/python.rs b/backend/parsers/windmill-parser-wac/src/python.rs new file mode 100644 index 0000000000..74f0117376 --- /dev/null +++ b/backend/parsers/windmill-parser-wac/src/python.rs @@ -0,0 +1,717 @@ +use std::collections::HashMap; + +use rustpython_parser::{ + ast::{ + Expr, ExprAwait, ExprCall, ExprName, Stmt, StmtExpr, StmtFor, StmtIf, StmtReturn, StmtTry, + StmtTryStar, StmtWhile, + }, + Parse, +}; + +use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag}; +use crate::validation::{self, CompileError}; + +struct LineIndex { + newline_offsets: Vec, +} + +impl LineIndex { + fn new(source: &str) -> Self { + let mut offsets = vec![0]; + for (i, c) in source.char_indices() { + if c == '\n' { + offsets.push(i + 1); + } + } + Self { newline_offsets: offsets } + } + + fn line_of(&self, byte_offset: usize) -> usize { + match self.newline_offsets.binary_search(&byte_offset) { + Ok(line) => line + 1, + Err(line) => line, + } + } +} + +/// Maps task function name → optional external path (from `@task(path="...")`) +type TaskFunctions = HashMap>; + +/// First pass: scan top-level `@task async def foo(...)` declarations. +fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { + let mut tasks = HashMap::new(); + for stmt in stmts { + if let Stmt::AsyncFunctionDef(func) = stmt { + for dec in &func.decorator_list { + match dec { + // @task (bare decorator) + Expr::Name(ExprName { id, .. }) if id.as_str() == "task" => { + tasks.insert(func.name.to_string(), None); + } + // @task(path="...") + Expr::Call(call) => { + if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() { + if id.as_str() == "task" { + let path = extract_task_path_kwarg(call); + tasks.insert(func.name.to_string(), path); + } + } + } + _ => {} + } + } + } + } + tasks +} + +/// Extract the `path=` keyword argument from a `@task(path="...")` call. +fn extract_task_path_kwarg(call: &ExprCall) -> Option { + for kw in &call.keywords { + if let Some(ref arg) = kw.arg { + if arg.as_str() == "path" { + if let Expr::Constant(c) = &kw.value { + if let rustpython_parser::ast::Constant::Str(s) = &c.value { + return Some(s.to_string()); + } + } + } + } + } + None +} + +struct WacWalker { + nodes: Vec, + edges: Vec, + errors: Vec, + node_counter: usize, + line_index: LineIndex, + task_functions: TaskFunctions, + in_try: bool, + in_while: bool, + in_nested_func: bool, + in_comprehension: bool, +} + +impl WacWalker { + fn new(source: &str, task_functions: TaskFunctions) -> Self { + Self { + nodes: Vec::new(), + edges: Vec::new(), + errors: Vec::new(), + node_counter: 0, + line_index: LineIndex::new(source), + task_functions, + in_try: false, + in_while: false, + in_nested_func: false, + in_comprehension: false, + } + } + + fn next_id(&mut self) -> String { + let id = format!("step_{}", self.node_counter); + self.node_counter += 1; + id + } + + fn add_node(&mut self, node: DagNode) -> String { + let id = node.id.clone(); + self.nodes.push(node); + id + } + + fn add_edge(&mut self, from: &str, to: &str, label: Option) { + self.edges + .push(DagEdge { from: from.to_string(), to: to.to_string(), label }); + } + + fn line_of_expr(&self, expr: &Expr) -> usize { + let offset = match expr { + Expr::Call(c) => c.range.start().to_usize(), + Expr::Await(a) => a.range.start().to_usize(), + Expr::Attribute(a) => a.range.start().to_usize(), + Expr::Name(n) => n.range.start().to_usize(), + _ => 0, + }; + self.line_index.line_of(offset) + } + + fn line_of_stmt(&self, stmt: &Stmt) -> usize { + let offset = match stmt { + Stmt::If(s) => s.range.start().to_usize(), + Stmt::For(s) => s.range.start().to_usize(), + Stmt::While(s) => s.range.start().to_usize(), + Stmt::Return(s) => s.range.start().to_usize(), + Stmt::Expr(s) => s.range.start().to_usize(), + Stmt::Try(s) => s.range.start().to_usize(), + Stmt::TryStar(s) => s.range.start().to_usize(), + Stmt::Assign(s) => s.range.start().to_usize(), + Stmt::AnnAssign(s) => s.range.start().to_usize(), + Stmt::FunctionDef(s) => s.range.start().to_usize(), + Stmt::AsyncFunctionDef(s) => s.range.start().to_usize(), + _ => 0, + }; + self.line_index.line_of(offset) + } + + /// Check if an expression is a call to a known @task function + fn is_task_fn_call(&self, expr: &Expr) -> bool { + if let Expr::Call(call) = expr { + if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() { + return self.task_functions.contains_key(id.as_str()); + } + } + false + } + + /// Check if an expression is `asyncio.gather(...)` call + fn is_asyncio_gather_call(expr: &Expr) -> bool { + if let Expr::Call(call) = expr { + if let Expr::Attribute(rustpython_parser::ast::ExprAttribute { value, attr, .. }) = + call.func.as_ref() + { + if attr.as_str() == "gather" { + if let Expr::Name(ExprName { id, .. }) = value.as_ref() { + return id.as_str() == "asyncio"; + } + } + } + } + false + } + + /// Extract step name and script from a task function call. + /// Name = function name, script = task_path or function name. + fn extract_step_info_from_task_call(&self, call: &ExprCall) -> Option<(String, String)> { + if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() { + let name = id.to_string(); + let script = self + .task_functions + .get(id.as_str()) + .and_then(|p| p.clone()) + .unwrap_or_else(|| name.clone()); + Some((name, script)) + } else { + None + } + } + + fn expr_to_source(expr: &Expr) -> String { + match expr { + Expr::Compare(c) => { + let left = Self::expr_to_source(&c.left); + if let Some(comparator) = c.comparators.first() { + let right = Self::expr_to_source(comparator); + let op = match c.ops.first() { + Some(rustpython_parser::ast::CmpOp::Gt) => ">", + Some(rustpython_parser::ast::CmpOp::Lt) => "<", + Some(rustpython_parser::ast::CmpOp::GtE) => ">=", + Some(rustpython_parser::ast::CmpOp::LtE) => "<=", + Some(rustpython_parser::ast::CmpOp::Eq) => "==", + Some(rustpython_parser::ast::CmpOp::NotEq) => "!=", + Some(rustpython_parser::ast::CmpOp::In) => "in", + Some(rustpython_parser::ast::CmpOp::NotIn) => "not in", + Some(rustpython_parser::ast::CmpOp::Is) => "is", + Some(rustpython_parser::ast::CmpOp::IsNot) => "is not", + None => "?", + }; + format!("{left} {op} {right}") + } else { + left + } + } + Expr::Subscript(s) => { + let value = Self::expr_to_source(&s.value); + let slice = Self::expr_to_source(&s.slice); + format!("{value}[{slice}]") + } + Expr::Attribute(a) => { + let value = Self::expr_to_source(&a.value); + format!("{value}.{}", a.attr) + } + Expr::Name(n) => n.id.to_string(), + Expr::Constant(c) => match &c.value { + rustpython_parser::ast::Constant::Str(s) => format!("\"{s}\""), + rustpython_parser::ast::Constant::Int(i) => i.to_string(), + rustpython_parser::ast::Constant::Float(f) => f.to_string(), + rustpython_parser::ast::Constant::Bool(b) => b.to_string(), + rustpython_parser::ast::Constant::None => "None".to_string(), + _ => "...".to_string(), + }, + _ => "...".to_string(), + } + } + + /// Check if a statement body contains any task function calls (recursively) + fn body_contains_step(&self, body: &[Stmt]) -> bool { + for stmt in body { + if self.stmt_contains_step(stmt) { + return true; + } + } + false + } + + fn stmt_contains_step(&self, stmt: &Stmt) -> bool { + match stmt { + Stmt::Expr(StmtExpr { value, .. }) => self.expr_contains_step(value), + Stmt::Assign(a) => self.expr_contains_step(&a.value), + Stmt::If(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse), + Stmt::For(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse), + Stmt::While(s) => { + self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse) + } + Stmt::Try(s) => { + self.body_contains_step(&s.body) + || self.body_contains_step(&s.orelse) + || self.body_contains_step(&s.finalbody) + || s.handlers.iter().any(|h| match h { + rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + self.body_contains_step(&eh.body) + } + }) + } + Stmt::TryStar(s) => { + self.body_contains_step(&s.body) + || self.body_contains_step(&s.orelse) + || self.body_contains_step(&s.finalbody) + || s.handlers.iter().any(|h| match h { + rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + self.body_contains_step(&eh.body) + } + }) + } + Stmt::Return(_) => false, + _ => false, + } + } + + fn expr_contains_step(&self, expr: &Expr) -> bool { + if self.is_task_fn_call(expr) { + return true; + } + match expr { + Expr::Await(ExprAwait { value, .. }) => self.expr_contains_step(value), + Expr::Call(call) => { + if self.is_task_fn_call(&Expr::Call(call.clone())) { + return true; + } + if Self::is_asyncio_gather_call(&Expr::Call(call.clone())) { + return call.args.iter().any(|a| self.expr_contains_step(a)); + } + false + } + _ => false, + } + } + + /// Walk a list of statements, returning (first_node_id, last_node_id) + fn walk_body(&mut self, body: &[Stmt]) -> Option<(String, String)> { + let mut first_id: Option = None; + let mut prev_id: Option = None; + + for stmt in body { + if let Some((stmt_first, stmt_last)) = self.walk_stmt(stmt) { + if let Some(ref prev) = prev_id { + self.add_edge(prev, &stmt_first, None); + } + if first_id.is_none() { + first_id = Some(stmt_first); + } + prev_id = Some(stmt_last); + } + } + + match (first_id, prev_id) { + (Some(f), Some(l)) => Some((f, l)), + _ => None, + } + } + + fn walk_stmt(&mut self, stmt: &Stmt) -> Option<(String, String)> { + match stmt { + Stmt::Expr(StmtExpr { value, .. }) => self.walk_expr_stmt(value), + Stmt::Assign(a) => self.walk_expr_stmt(&a.value), + Stmt::If(if_stmt) => self.walk_if(if_stmt), + Stmt::For(for_stmt) => self.walk_for(for_stmt), + Stmt::While(while_stmt) => self.walk_while(while_stmt), + Stmt::Try(try_stmt) => self.walk_try(try_stmt), + Stmt::TryStar(try_stmt) => self.walk_try_star(try_stmt), + Stmt::Return(ret) => self.walk_return(ret), + Stmt::FunctionDef(_) | Stmt::AsyncFunctionDef(_) => { + if self.stmt_contains_step(stmt) { + self.errors.push(validation::error_step_in_nested_function( + self.line_of_stmt(stmt), + )); + } + None + } + _ => None, + } + } + + fn walk_expr_stmt(&mut self, expr: &Expr) -> Option<(String, String)> { + // await task_fn(...) + if let Expr::Await(ExprAwait { value, .. }) = expr { + // await task_fn(...) + if let Expr::Call(call) = value.as_ref() { + if self.is_task_fn_call(&Expr::Call(call.clone())) { + return self.emit_step(call, expr); + } + } + // await asyncio.gather(task_fn(...), task_fn(...), ...) + if Self::is_asyncio_gather_call(value) { + if let Expr::Call(gather_call) = value.as_ref() { + return self.emit_parallel(gather_call, expr); + } + } + } + + // Bare task_fn() without await — validation error + if self.is_task_fn_call(expr) { + self.errors + .push(validation::error_missing_await(self.line_of_expr(expr))); + } + + None + } + + fn emit_step(&mut self, call: &ExprCall, expr: &Expr) -> Option<(String, String)> { + if self.in_try { + self.errors + .push(validation::error_step_in_try(self.line_of_expr(expr))); + return None; + } + if self.in_while { + self.errors + .push(validation::error_step_in_while(self.line_of_expr(expr))); + return None; + } + if self.in_nested_func { + self.errors.push(validation::error_step_in_nested_function( + self.line_of_expr(expr), + )); + return None; + } + if self.in_comprehension { + self.errors.push(validation::error_step_in_comprehension( + self.line_of_expr(expr), + )); + return None; + } + + let (name, script) = self + .extract_step_info_from_task_call(call) + .unwrap_or(("unknown".into(), "unknown".into())); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::Step { name: name.clone(), script }, + label: name, + line: self.line_of_expr(expr), + }); + Some((node_id.clone(), node_id)) + } + + fn emit_parallel(&mut self, gather_call: &ExprCall, expr: &Expr) -> Option<(String, String)> { + if self.in_try { + self.errors + .push(validation::error_step_in_try(self.line_of_expr(expr))); + return None; + } + if self.in_while { + self.errors + .push(validation::error_step_in_while(self.line_of_expr(expr))); + return None; + } + + let line = self.line_of_expr(expr); + let start_id = self.next_id(); + let start_node_id = self.add_node(DagNode { + id: start_id.clone(), + node_type: DagNodeType::ParallelStart, + label: "parallel".to_string(), + line, + }); + + let mut step_ids = Vec::new(); + for arg in &gather_call.args { + // Each arg should be task_fn(...) + if let Expr::Call(call) = arg { + if self.is_task_fn_call(&Expr::Call(call.clone())) { + let (name, script) = self + .extract_step_info_from_task_call(call) + .unwrap_or(("unknown".into(), "unknown".into())); + let step_id = self.next_id(); + let node_id = self.add_node(DagNode { + id: step_id.clone(), + node_type: DagNodeType::Step { name: name.clone(), script }, + label: name, + line: self.line_of_expr(arg), + }); + self.add_edge(&start_node_id, &node_id, None); + step_ids.push(node_id); + } + } + } + + let end_id = self.next_id(); + let end_node_id = self.add_node(DagNode { + id: end_id.clone(), + node_type: DagNodeType::ParallelEnd, + label: "join".to_string(), + line, + }); + + for step_id in &step_ids { + self.add_edge(step_id, &end_node_id, None); + } + + Some((start_node_id, end_node_id)) + } + + fn walk_if(&mut self, if_stmt: &StmtIf) -> Option<(String, String)> { + let has_steps_in_body = self.body_contains_step(&if_stmt.body); + let has_steps_in_else = self.body_contains_step(&if_stmt.orelse); + + if !has_steps_in_body && !has_steps_in_else { + return None; + } + + let line = self.line_index.line_of(if_stmt.range.start().to_usize()); + let condition_source = Self::expr_to_source(&if_stmt.test); + + let branch_id = self.next_id(); + let branch_node_id = self.add_node(DagNode { + id: branch_id.clone(), + node_type: DagNodeType::Branch { condition_source }, + label: "if".to_string(), + line, + }); + + let merge_id = format!("{branch_id}_merge"); + + let mut last_ids = Vec::new(); + + if let Some((true_first, true_last)) = self.walk_body(&if_stmt.body) { + self.add_edge(&branch_node_id, &true_first, Some("true".to_string())); + last_ids.push(true_last); + } else { + last_ids.push(branch_node_id.clone()); + } + + if !if_stmt.orelse.is_empty() { + if let Some((else_first, else_last)) = self.walk_body(&if_stmt.orelse) { + self.add_edge(&branch_node_id, &else_first, Some("false".to_string())); + last_ids.push(else_last); + } else { + last_ids.push(branch_node_id.clone()); + } + } + + if last_ids.len() == 1 { + Some((branch_node_id, last_ids.into_iter().next().unwrap())) + } else { + Some((branch_node_id, merge_id)) + } + } + + fn walk_for(&mut self, for_stmt: &StmtFor) -> Option<(String, String)> { + if !self.body_contains_step(&for_stmt.body) { + return None; + } + + let line = self.line_index.line_of(for_stmt.range.start().to_usize()); + let iter_source = Self::expr_to_source(&for_stmt.iter); + + let start_id = self.next_id(); + let start_node_id = self.add_node(DagNode { + id: start_id.clone(), + node_type: DagNodeType::LoopStart { iter_source }, + label: "for".to_string(), + line, + }); + + if let Some((body_first, body_last)) = self.walk_body(&for_stmt.body) { + self.add_edge(&start_node_id, &body_first, None); + self.add_edge(&body_last, &start_node_id, Some("next".to_string())); + } + + let end_id = self.next_id(); + let end_node_id = self.add_node(DagNode { + id: end_id.clone(), + node_type: DagNodeType::LoopEnd, + label: "end for".to_string(), + line, + }); + self.add_edge(&start_node_id, &end_node_id, Some("done".to_string())); + + Some((start_node_id, end_node_id)) + } + + fn walk_while(&mut self, while_stmt: &StmtWhile) -> Option<(String, String)> { + if self.body_contains_step(&while_stmt.body) { + let line = self.line_index.line_of(while_stmt.range.start().to_usize()); + self.errors.push(validation::error_step_in_while(line)); + } + None + } + + fn walk_try(&mut self, try_stmt: &StmtTry) -> Option<(String, String)> { + let has_steps = self.body_contains_step(&try_stmt.body) + || self.body_contains_step(&try_stmt.orelse) + || self.body_contains_step(&try_stmt.finalbody) + || try_stmt.handlers.iter().any(|h| match h { + rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + self.body_contains_step(&eh.body) + } + }); + + if has_steps { + let line = self.line_index.line_of(try_stmt.range.start().to_usize()); + self.errors.push(validation::error_step_in_try(line)); + } + None + } + + fn walk_try_star(&mut self, try_stmt: &StmtTryStar) -> Option<(String, String)> { + let has_steps = self.body_contains_step(&try_stmt.body) + || self.body_contains_step(&try_stmt.orelse) + || self.body_contains_step(&try_stmt.finalbody) + || try_stmt.handlers.iter().any(|h| match h { + rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + self.body_contains_step(&eh.body) + } + }); + + if has_steps { + let line = self.line_index.line_of(try_stmt.range.start().to_usize()); + self.errors.push(validation::error_step_in_try(line)); + } + None + } + + fn walk_return(&mut self, ret: &StmtReturn) -> Option<(String, String)> { + let line = self.line_index.line_of(ret.range.start().to_usize()); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::Return, + label: "return".to_string(), + line, + }); + Some((node_id.clone(), node_id)) + } +} + +/// Extract workflow function parameters (no longer skips ctx) +fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec { + let mut params = Vec::new(); + for arg_with_default in args.args.iter().chain(args.posonlyargs.iter()) { + let name = arg_with_default.def.arg.to_string(); + let typ = arg_with_default + .def + .annotation + .as_ref() + .map(|ann| WacWalker::expr_to_source(ann)); + params.push(Param { name, typ }); + } + params +} + +pub fn parse_python_workflow(code: &str) -> Result> { + let ast = rustpython_parser::ast::Suite::parse(code, "") + .map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?; + + // First pass: collect @task functions + let task_functions = collect_task_functions(&ast); + + // Find the @workflow async def + let workflow_fn = ast.iter().find_map(|stmt| { + if let Stmt::AsyncFunctionDef(func) = stmt { + let has_workflow_decorator = func.decorator_list.iter().any(|dec| { + if let Expr::Name(ExprName { id, .. }) = dec { + id.as_str() == "workflow" + } else { + false + } + }); + if has_workflow_decorator { + return Some(func); + } + } + // Also check non-async for error reporting + if let Stmt::FunctionDef(func) = stmt { + let has_workflow_decorator = func.decorator_list.iter().any(|dec| { + if let Expr::Name(ExprName { id, .. }) = dec { + id.as_str() == "workflow" + } else { + false + } + }); + if has_workflow_decorator { + return None; // Will be reported as not-async below + } + } + None + }); + + // Check for non-async workflow function + let non_async_workflow = ast.iter().find_map(|stmt| { + if let Stmt::FunctionDef(func) = stmt { + let has_workflow_decorator = func.decorator_list.iter().any(|dec| { + if let Expr::Name(ExprName { id, .. }) = dec { + id.as_str() == "workflow" + } else { + false + } + }); + if has_workflow_decorator { + let line_index = LineIndex::new(code); + return Some(line_index.line_of(func.range.start().to_usize())); + } + } + None + }); + + if let Some(line) = non_async_workflow { + if workflow_fn.is_none() { + return Err(vec![validation::error_not_async(line)]); + } + } + + let workflow_fn = workflow_fn.ok_or_else(|| { + vec![CompileError { message: "No @workflow async function found.".to_string(), line: 0 }] + })?; + + let params = extract_params(&workflow_fn.args); + let source_hash = compute_source_hash(code); + + let mut walker = WacWalker::new(code, task_functions); + walker.walk_body(&workflow_fn.body); + + if !walker.errors.is_empty() { + return Err(walker.errors); + } + + Ok(WorkflowDag { nodes: walker.nodes, edges: walker.edges, params, source_hash }) +} + +fn compute_source_hash(code: &str) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(code.as_bytes()); + format!("{:x}", hasher.finalize()) +} + +trait ToUsize { + fn to_usize(self) -> usize; +} + +impl ToUsize for rustpython_parser::text_size::TextSize { + fn to_usize(self) -> usize { + u32::from(self) as usize + } +} diff --git a/backend/parsers/windmill-parser-wac/src/typescript.rs b/backend/parsers/windmill-parser-wac/src/typescript.rs new file mode 100644 index 0000000000..bd777749ea --- /dev/null +++ b/backend/parsers/windmill-parser-wac/src/typescript.rs @@ -0,0 +1,739 @@ +use std::collections::HashMap; + +use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Spanned}; +use swc_ecma_ast::*; +use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsSyntax}; + +use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag}; +use crate::validation::{self, CompileError}; + +/// Maps task function name → optional external path (from `task("f/path", ...)`) +type TaskFunctions = HashMap>; + +/// First pass: scan top-level `const foo = task(async (...) => {})` or +/// `const foo = task("f/path", async (...) => {})` declarations. +fn collect_task_functions(module: &Module) -> TaskFunctions { + let mut tasks = HashMap::new(); + for item in &module.body { + // const foo = task(async (...) => { ... }) + // const foo = task("f/path", async (...) => { ... }) + if let ModuleItem::Stmt(Stmt::Decl(Decl::Var(var_decl))) = item { + for decl in &var_decl.decls { + if let (Some(name), Some(init)) = (extract_var_name(&decl.name), &decl.init) { + if let Some(path) = extract_task_call_info(init) { + tasks.insert(name, path); + } + } + } + } + // export const foo = task(...) + if let ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(export)) = item { + if let Decl::Var(var_decl) = &export.decl { + for decl in &var_decl.decls { + if let (Some(name), Some(init)) = (extract_var_name(&decl.name), &decl.init) { + if let Some(path) = extract_task_call_info(init) { + tasks.insert(name, path); + } + } + } + } + } + } + tasks +} + +/// Extract variable name from a pattern (simple ident case) +fn extract_var_name(pat: &Pat) -> Option { + if let Pat::Ident(BindingIdent { id, .. }) = pat { + Some(id.sym.to_string()) + } else { + None + } +} + +/// Check if expr is `task(async fn)` or `task("path", async fn)`. +/// Returns Some(optional_path) if it is a task() call. +fn extract_task_call_info(expr: &Expr) -> Option> { + if let Expr::Call(call) = expr { + if let Callee::Expr(callee) = &call.callee { + if let Expr::Ident(ident) = callee.as_ref() { + if ident.sym.as_ref() == "task" { + // task("f/path", async fn) or task(async fn) + if call.args.len() == 2 { + // task("f/path", async fn) + let path = extract_string_lit(&call.args[0].expr); + return Some(path); + } else if call.args.len() == 1 { + // task(async fn) + return Some(None); + } + } + } + } + } + None +} + +struct TsWacWalker { + nodes: Vec, + edges: Vec, + errors: Vec, + node_counter: usize, + cm: Lrc, + task_functions: TaskFunctions, + in_try: bool, + in_while: bool, + in_nested_func: bool, +} + +impl TsWacWalker { + fn new(cm: Lrc, task_functions: TaskFunctions) -> Self { + Self { + nodes: Vec::new(), + edges: Vec::new(), + errors: Vec::new(), + node_counter: 0, + cm, + task_functions, + in_try: false, + in_while: false, + in_nested_func: false, + } + } + + fn next_id(&mut self) -> String { + let id = format!("step_{}", self.node_counter); + self.node_counter += 1; + id + } + + fn add_node(&mut self, node: DagNode) -> String { + let id = node.id.clone(); + self.nodes.push(node); + id + } + + fn add_edge(&mut self, from: &str, to: &str, label: Option) { + self.edges + .push(DagEdge { from: from.to_string(), to: to.to_string(), label }); + } + + fn span_line(&self, span: swc_common::Span) -> usize { + let loc = self.cm.lookup_char_pos(span.lo); + loc.line + } + + /// Check if expr is a call to a known task function + fn is_task_call(&self, expr: &Expr) -> bool { + if let Expr::Call(call) = expr { + if let Callee::Expr(callee) = &call.callee { + if let Expr::Ident(ident) = callee.as_ref() { + return self.task_functions.contains_key(ident.sym.as_ref()); + } + } + } + false + } + + /// Check if expr is `Promise.all([...])` + fn is_promise_all(expr: &Expr) -> bool { + if let Expr::Call(call) = expr { + if let Callee::Expr(callee) = &call.callee { + if let Expr::Member(MemberExpr { obj, prop: MemberProp::Ident(prop), .. }) = + callee.as_ref() + { + if prop.sym.as_ref() == "all" { + if let Expr::Ident(ident) = obj.as_ref() { + return ident.sym.as_ref() == "Promise"; + } + } + } + } + } + false + } + + /// Extract step name and script from a task function call. + /// Name = function name, script = task_path or function name. + fn extract_step_info_from_task_call(&self, call: &CallExpr) -> Option<(String, String)> { + if let Callee::Expr(callee) = &call.callee { + if let Expr::Ident(ident) = callee.as_ref() { + let name = ident.sym.to_string(); + let script = self + .task_functions + .get(ident.sym.as_ref()) + .and_then(|p| p.clone()) + .unwrap_or_else(|| name.clone()); + return Some((name, script)); + } + } + None + } + + fn expr_to_source(&self, expr: &Expr) -> String { + let span = expr.span(); + self.cm + .span_to_snippet(span) + .unwrap_or_else(|_| "...".to_string()) + } + + fn body_contains_step(&self, stmts: &[Stmt]) -> bool { + stmts.iter().any(|s| self.stmt_contains_step(s)) + } + + fn stmt_contains_step(&self, stmt: &Stmt) -> bool { + match stmt { + Stmt::Expr(expr_stmt) => self.expr_contains_step(&expr_stmt.expr), + Stmt::Decl(Decl::Var(var_decl)) => var_decl.decls.iter().any(|d| { + d.init + .as_ref() + .map_or(false, |init| self.expr_contains_step(init)) + }), + Stmt::If(if_stmt) => { + self.stmt_contains_step(&if_stmt.cons) + || if_stmt + .alt + .as_ref() + .map_or(false, |alt| self.stmt_contains_step(alt)) + } + Stmt::Block(block) => self.body_contains_step(&block.stmts), + Stmt::For(for_stmt) => self.stmt_contains_step(&for_stmt.body), + Stmt::ForIn(for_in) => self.stmt_contains_step(&for_in.body), + Stmt::ForOf(for_of) => self.stmt_contains_step(&for_of.body), + Stmt::While(while_stmt) => self.stmt_contains_step(&while_stmt.body), + Stmt::Try(try_stmt) => { + self.body_contains_step(&try_stmt.block.stmts) + || try_stmt + .handler + .as_ref() + .map_or(false, |h| self.body_contains_step(&h.body.stmts)) + || try_stmt + .finalizer + .as_ref() + .map_or(false, |f| self.body_contains_step(&f.stmts)) + } + Stmt::Return(ret) => ret + .arg + .as_ref() + .map_or(false, |arg| self.expr_contains_step(arg)), + _ => false, + } + } + + fn expr_contains_step(&self, expr: &Expr) -> bool { + if self.is_task_call(expr) { + return true; + } + match expr { + Expr::Await(await_expr) => self.expr_contains_step(&await_expr.arg), + Expr::Call(call) => { + if Self::is_promise_all(&Expr::Call(call.clone())) { + return call.args.iter().any(|a| self.expr_contains_step(&a.expr)); + } + false + } + Expr::Paren(p) => self.expr_contains_step(&p.expr), + _ => false, + } + } + + fn walk_body(&mut self, stmts: &[Stmt]) -> Option<(String, String)> { + let mut first_id: Option = None; + let mut prev_id: Option = None; + + for stmt in stmts { + if let Some((stmt_first, stmt_last)) = self.walk_stmt(stmt) { + if let Some(ref prev) = prev_id { + self.add_edge(prev, &stmt_first, None); + } + if first_id.is_none() { + first_id = Some(stmt_first); + } + prev_id = Some(stmt_last); + } + } + + match (first_id, prev_id) { + (Some(f), Some(l)) => Some((f, l)), + _ => None, + } + } + + fn walk_stmt(&mut self, stmt: &Stmt) -> Option<(String, String)> { + match stmt { + Stmt::Expr(expr_stmt) => self.walk_expr_stmt(&expr_stmt.expr), + Stmt::Decl(Decl::Var(var_decl)) => { + // const result = await task_fn(...) + for decl in &var_decl.decls { + if let Some(init) = &decl.init { + if let Some(result) = self.walk_expr_stmt(init) { + return Some(result); + } + } + } + None + } + Stmt::If(if_stmt) => self.walk_if(if_stmt), + Stmt::For(for_stmt) => self.walk_for_stmt(for_stmt), + Stmt::ForIn(for_in) => self.walk_for_in(for_in), + Stmt::ForOf(for_of) => self.walk_for_of(for_of), + Stmt::While(while_stmt) => self.walk_while(while_stmt), + Stmt::Try(try_stmt) => self.walk_try(try_stmt), + Stmt::Block(block) => self.walk_body(&block.stmts), + Stmt::Return(ret) => self.walk_return(ret), + Stmt::Decl(Decl::Fn(_)) => { + if self.stmt_contains_step(stmt) { + self.errors.push(validation::error_step_in_nested_function( + self.span_line(stmt.span()), + )); + } + None + } + _ => None, + } + } + + fn walk_expr_stmt(&mut self, expr: &Expr) -> Option<(String, String)> { + // await task_fn(...) + if let Expr::Await(await_expr) = expr { + if let Expr::Call(call) = await_expr.arg.as_ref() { + if self.is_task_call(&Expr::Call(call.clone())) { + return self.emit_step(call, expr); + } + } + // await Promise.all([task_fn(...), ...]) + if Self::is_promise_all(&await_expr.arg) { + if let Expr::Call(promise_call) = await_expr.arg.as_ref() { + return self.emit_parallel(promise_call, expr); + } + } + } + + // Bare task_fn() without await + if self.is_task_call(expr) { + self.errors + .push(validation::error_missing_await(self.span_line(expr.span()))); + } + + None + } + + fn emit_step(&mut self, call: &CallExpr, expr: &Expr) -> Option<(String, String)> { + if self.in_try { + self.errors + .push(validation::error_step_in_catch(self.span_line(expr.span()))); + return None; + } + if self.in_while { + self.errors + .push(validation::error_step_in_while(self.span_line(expr.span()))); + return None; + } + if self.in_nested_func { + self.errors.push(validation::error_step_in_nested_function( + self.span_line(expr.span()), + )); + return None; + } + + let (name, script) = self + .extract_step_info_from_task_call(call) + .unwrap_or(("unknown".into(), "unknown".into())); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::Step { name: name.clone(), script }, + label: name, + line: self.span_line(expr.span()), + }); + Some((node_id.clone(), node_id)) + } + + fn emit_parallel(&mut self, promise_call: &CallExpr, expr: &Expr) -> Option<(String, String)> { + if self.in_try { + self.errors + .push(validation::error_step_in_catch(self.span_line(expr.span()))); + return None; + } + if self.in_while { + self.errors + .push(validation::error_step_in_while(self.span_line(expr.span()))); + return None; + } + + let line = self.span_line(expr.span()); + let start_id = self.next_id(); + let start_node_id = self.add_node(DagNode { + id: start_id.clone(), + node_type: DagNodeType::ParallelStart, + label: "parallel".to_string(), + line, + }); + + let mut step_ids = Vec::new(); + + // Promise.all takes an array as first argument + if let Some(first_arg) = promise_call.args.first() { + if let Expr::Array(ArrayLit { elems, .. }) = first_arg.expr.as_ref() { + for elem in elems.iter().flatten() { + if let Expr::Call(call) = elem.expr.as_ref() { + if self.is_task_call(&Expr::Call(call.clone())) { + let (name, script) = self + .extract_step_info_from_task_call(call) + .unwrap_or(("unknown".into(), "unknown".into())); + let step_id = self.next_id(); + let node_id = self.add_node(DagNode { + id: step_id.clone(), + node_type: DagNodeType::Step { name: name.clone(), script }, + label: name, + line: self.span_line(elem.expr.span()), + }); + self.add_edge(&start_node_id, &node_id, None); + step_ids.push(node_id); + } + } + } + } + } + + let end_id = self.next_id(); + let end_node_id = self.add_node(DagNode { + id: end_id.clone(), + node_type: DagNodeType::ParallelEnd, + label: "join".to_string(), + line, + }); + + for step_id in &step_ids { + self.add_edge(step_id, &end_node_id, None); + } + + Some((start_node_id, end_node_id)) + } + + fn walk_if(&mut self, if_stmt: &IfStmt) -> Option<(String, String)> { + let has_steps_cons = self.stmt_contains_step(&if_stmt.cons); + let has_steps_alt = if_stmt + .alt + .as_ref() + .map_or(false, |a| self.stmt_contains_step(a)); + + if !has_steps_cons && !has_steps_alt { + return None; + } + + let line = self.span_line(if_stmt.span); + let condition_source = self.expr_to_source(&if_stmt.test); + + let branch_id = self.next_id(); + let branch_node_id = self.add_node(DagNode { + id: branch_id.clone(), + node_type: DagNodeType::Branch { condition_source }, + label: "if".to_string(), + line, + }); + + let mut last_ids = Vec::new(); + + // True branch + if let Some((true_first, true_last)) = self.walk_stmt(&if_stmt.cons) { + self.add_edge(&branch_node_id, &true_first, Some("true".to_string())); + last_ids.push(true_last); + } else { + last_ids.push(branch_node_id.clone()); + } + + // False branch + if let Some(alt) = &if_stmt.alt { + if let Some((else_first, else_last)) = self.walk_stmt(alt) { + self.add_edge(&branch_node_id, &else_first, Some("false".to_string())); + last_ids.push(else_last); + } else { + last_ids.push(branch_node_id.clone()); + } + } + + if last_ids.len() == 1 { + Some((branch_node_id, last_ids.into_iter().next().unwrap())) + } else { + let merge_id = format!("{branch_id}_merge"); + Some((branch_node_id, merge_id)) + } + } + + fn walk_for_stmt(&mut self, for_stmt: &ForStmt) -> Option<(String, String)> { + if !self.stmt_contains_step(&for_stmt.body) { + return None; + } + self.walk_loop_body(&for_stmt.body, for_stmt.span, "for") + } + + fn walk_for_in(&mut self, for_in: &ForInStmt) -> Option<(String, String)> { + if !self.stmt_contains_step(&for_in.body) { + return None; + } + let iter_source = self.expr_to_source(&for_in.right); + self.walk_loop_body_with_iter(&for_in.body, for_in.span, &iter_source) + } + + fn walk_for_of(&mut self, for_of: &ForOfStmt) -> Option<(String, String)> { + if !self.stmt_contains_step(&for_of.body) { + return None; + } + let iter_source = self.expr_to_source(&for_of.right); + self.walk_loop_body_with_iter(&for_of.body, for_of.span, &iter_source) + } + + fn walk_loop_body( + &mut self, + body: &Stmt, + span: swc_common::Span, + _label: &str, + ) -> Option<(String, String)> { + self.walk_loop_body_with_iter(body, span, "...") + } + + fn walk_loop_body_with_iter( + &mut self, + body: &Stmt, + span: swc_common::Span, + iter_source: &str, + ) -> Option<(String, String)> { + let line = self.span_line(span); + let start_id = self.next_id(); + let start_node_id = self.add_node(DagNode { + id: start_id.clone(), + node_type: DagNodeType::LoopStart { iter_source: iter_source.to_string() }, + label: "for".to_string(), + line, + }); + + if let Some((body_first, body_last)) = self.walk_stmt(body) { + self.add_edge(&start_node_id, &body_first, None); + self.add_edge(&body_last, &start_node_id, Some("next".to_string())); + } + + let end_id = self.next_id(); + let end_node_id = self.add_node(DagNode { + id: end_id.clone(), + node_type: DagNodeType::LoopEnd, + label: "end for".to_string(), + line, + }); + self.add_edge(&start_node_id, &end_node_id, Some("done".to_string())); + + Some((start_node_id, end_node_id)) + } + + fn walk_while(&mut self, while_stmt: &WhileStmt) -> Option<(String, String)> { + if self.stmt_contains_step(&while_stmt.body) { + self.errors.push(validation::error_step_in_while( + self.span_line(while_stmt.span), + )); + } + None + } + + fn walk_try(&mut self, try_stmt: &TryStmt) -> Option<(String, String)> { + let has_steps = self.body_contains_step(&try_stmt.block.stmts) + || try_stmt + .handler + .as_ref() + .map_or(false, |h| self.body_contains_step(&h.body.stmts)) + || try_stmt + .finalizer + .as_ref() + .map_or(false, |f| self.body_contains_step(&f.stmts)); + + if has_steps { + self.errors.push(validation::error_step_in_catch( + self.span_line(try_stmt.span), + )); + } + None + } + + fn walk_return(&mut self, ret: &ReturnStmt) -> Option<(String, String)> { + let line = self.span_line(ret.span); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::Return, + label: "return".to_string(), + line, + }); + Some((node_id.clone(), node_id)) + } +} + +/// Extract workflow function params (no longer skips ctx) +fn extract_ts_params(params: &[swc_ecma_ast::Param], cm: &Lrc) -> Vec { + let mut result = Vec::new(); + for param in params { + let (name, typ) = match ¶m.pat { + Pat::Ident(BindingIdent { id, type_ann, .. }) => { + let name = id.sym.to_string(); + let typ = type_ann.as_ref().map(|ann| { + cm.span_to_snippet(ann.type_ann.span()) + .unwrap_or_else(|_| "unknown".to_string()) + }); + (name, typ) + } + _ => continue, + }; + result.push(Param { name, typ }); + } + result +} + +pub fn parse_ts_workflow(code: &str) -> Result> { + let cm: Lrc = Default::default(); + let fm = cm.new_source_file(FileName::Custom("workflow.ts".into()).into(), code.into()); + let lexer = Lexer::new( + Syntax::Typescript(TsSyntax::default()), + Default::default(), + StringInput::from(&*fm), + None, + ); + + let mut parser = Parser::new_from(lexer); + let module = parser + .parse_module() + .map_err(|e| vec![CompileError { message: format!("Parse error: {e:?}"), line: 0 }])?; + + // First pass: collect task functions + let task_functions = collect_task_functions(&module); + + // Find: export default workflow(async (...) => { ... }) + // or: export default workflow(async function(...) { ... }) + let mut workflow_body: Option<(&[Stmt], Vec)> = None; + + for item in &module.body { + // export default workflow(async (...) => { ... }) + if let ModuleItem::ModuleDecl(ModuleDecl::ExportDefaultExpr(export)) = item { + if let Some(result) = find_workflow_call(&export.expr, &cm) { + workflow_body = Some(result); + break; + } + } + // const wf = workflow(async (...) => { ... }); export default wf; + if let ModuleItem::ModuleDecl(ModuleDecl::ExportDefaultDecl(export)) = item { + if let DefaultDecl::Fn(_) = &export.decl { + // `export default async function(...) { ... }` — not wrapped in workflow(), skip + } + } + if let ModuleItem::Stmt(Stmt::Decl(Decl::Var(var_decl))) = item { + for decl in &var_decl.decls { + if let Some(init) = &decl.init { + if let Some(result) = find_workflow_call(init, &cm) { + workflow_body = Some(result); + break; + } + } + } + } + } + + let (stmts, params) = workflow_body.ok_or_else(|| { + vec![CompileError { + message: "No workflow() wrapped async function found.".to_string(), + line: 0, + }] + })?; + + let source_hash = compute_source_hash(code); + + let mut walker = TsWacWalker::new(cm, task_functions); + walker.walk_body(stmts); + + if !walker.errors.is_empty() { + return Err(walker.errors); + } + + Ok(WorkflowDag { nodes: walker.nodes, edges: walker.edges, params, source_hash }) +} + +/// Find workflow(async (...) => { ... }) or workflow(async function(...) { ... }) +fn find_workflow_call<'a>(expr: &'a Expr, cm: &Lrc) -> Option<(&'a [Stmt], Vec)> { + if let Expr::Call(call) = expr { + // Check if callee is `workflow` + let is_workflow = match &call.callee { + Callee::Expr(callee_expr) => { + if let Expr::Ident(ident) = callee_expr.as_ref() { + ident.sym.as_ref() == "workflow" + } else { + false + } + } + _ => false, + }; + + if is_workflow { + if let Some(first_arg) = call.args.first() { + return extract_async_fn_body(&first_arg.expr, cm); + } + } + } + None +} + +fn extract_async_fn_body<'a>( + expr: &'a Expr, + cm: &Lrc, +) -> Option<(&'a [Stmt], Vec)> { + match expr { + Expr::Arrow(arrow) if arrow.is_async => { + let params = extract_arrow_params(&arrow.params, cm); + match &*arrow.body { + BlockStmtOrExpr::BlockStmt(block) => Some((&block.stmts, params)), + _ => None, + } + } + Expr::Fn(fn_expr) if fn_expr.function.is_async => { + let params = extract_ts_params(&fn_expr.function.params, cm); + fn_expr + .function + .body + .as_ref() + .map(|body| (body.stmts.as_slice(), params)) + } + Expr::Paren(p) => extract_async_fn_body(&p.expr, cm), + _ => None, + } +} + +/// Extract arrow function params (no longer skips ctx) +fn extract_arrow_params(pats: &[Pat], cm: &Lrc) -> Vec { + let mut result = Vec::new(); + for pat in pats { + match pat { + Pat::Ident(BindingIdent { id, type_ann, .. }) => { + let name = id.sym.to_string(); + let typ = type_ann.as_ref().map(|ann| { + cm.span_to_snippet(ann.type_ann.span()) + .unwrap_or_else(|_| "unknown".to_string()) + }); + result.push(Param { name, typ }); + } + _ => {} + } + } + result +} + +fn extract_string_lit(expr: &Expr) -> Option { + match expr { + Expr::Lit(Lit::Str(s)) => Some(s.value.to_string()), + Expr::Tpl(tpl) if tpl.exprs.is_empty() && tpl.quasis.len() == 1 => { + tpl.quasis.first().map(|q| q.raw.to_string()) + } + _ => None, + } +} + +fn compute_source_hash(code: &str) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(code.as_bytes()); + format!("{:x}", hasher.finalize()) +} diff --git a/backend/parsers/windmill-parser-wac/src/validation.rs b/backend/parsers/windmill-parser-wac/src/validation.rs new file mode 100644 index 0000000000..e3a57c6811 --- /dev/null +++ b/backend/parsers/windmill-parser-wac/src/validation.rs @@ -0,0 +1,64 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct CompileError { + pub message: String, + pub line: usize, +} + +impl std::fmt::Display for CompileError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "line {}: {}", self.line, self.message) + } +} + +pub fn error_step_in_try(line: usize) -> CompileError { + CompileError { + message: + "Task calls inside try/except are not allowed. Steps have built-in error handling." + .to_string(), + line, + } +} + +pub fn error_step_in_while(line: usize) -> CompileError { + CompileError { + message: "Task calls inside while loops are not allowed. Use for loops instead." + .to_string(), + line, + } +} + +pub fn error_step_in_nested_function(line: usize) -> CompileError { + CompileError { + message: "Task calls inside nested functions, closures, or lambdas are not allowed." + .to_string(), + line, + } +} + +pub fn error_step_in_comprehension(line: usize) -> CompileError { + CompileError { message: "Task calls inside comprehensions are not allowed.".to_string(), line } +} + +pub fn error_not_async(line: usize) -> CompileError { + CompileError { message: "Workflow function must be async.".to_string(), line } +} + +pub fn error_missing_await(line: usize) -> CompileError { + CompileError { + message: + "Task calls must be awaited directly or used inside asyncio.gather()/Promise.all()." + .to_string(), + line, + } +} + +pub fn error_step_in_catch(line: usize) -> CompileError { + CompileError { + message: + "Task calls inside catch blocks are not allowed. Steps have built-in error handling." + .to_string(), + line, + } +} diff --git a/backend/parsers/windmill-parser-wac/tests/python_tests.rs b/backend/parsers/windmill-parser-wac/tests/python_tests.rs new file mode 100644 index 0000000000..59f0b59f5c --- /dev/null +++ b/backend/parsers/windmill-parser-wac/tests/python_tests.rs @@ -0,0 +1,266 @@ +use windmill_parser_wac::dag::DagNodeType; +use windmill_parser_wac::python::parse_python_workflow; + +#[test] +fn test_simple_sequential_workflow() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def extract_data(url: str): ... +@task +async def load_data(data: list): ... + +@workflow +async def my_etl(url: str): + raw = await extract_data(url=url) + await load_data(data=raw) + return {"status": "done"} +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 3); // 2 steps + 1 return + assert_eq!(dag.edges.len(), 2); // step0->step1, step1->return + + // Check params (url — no ctx to skip) + assert_eq!(dag.params.len(), 1); + assert_eq!(dag.params[0].name, "url"); + assert_eq!(dag.params[0].typ.as_deref(), Some("str")); + + // Check first step + match &dag.nodes[0].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "extract_data"); + assert_eq!(script, "extract_data"); + } + _ => panic!("expected Step node"), + } + + // Check second step + match &dag.nodes[1].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "load_data"); + assert_eq!(script, "load_data"); + } + _ => panic!("expected Step node"), + } + + // Check return + assert!(matches!(dag.nodes[2].node_type, DagNodeType::Return)); + + // Check source hash is non-empty + assert!(!dag.source_hash.is_empty()); +} + +#[test] +fn test_parallel_workflow() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def extract_data(url: str): ... +@task +async def clean_data(data: list): ... +@task +async def compute_stats(data: list): ... +@task +async def load_to_warehouse(rows: list): ... + +@workflow +async def my_etl(url: str): + raw = await extract_data(url=url) + cleaned, stats = await asyncio.gather( + clean_data(data=raw), + compute_stats(data=raw), + ) + await load_to_warehouse(rows=cleaned) + return {"status": "done"} +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + + // extract, ParallelStart, clean, stats, ParallelEnd, load, return = 7 + assert_eq!(dag.nodes.len(), 7); + + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::ParallelStart)); + assert!(matches!(dag.nodes[2].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[3].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[4].node_type, DagNodeType::ParallelEnd)); + assert!(matches!(dag.nodes[5].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[6].node_type, DagNodeType::Return)); +} + +#[test] +fn test_conditional_workflow() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def send_alert(msg: str): ... +@task +async def load_data(): ... + +@workflow +async def my_etl(count: int): + if count > 100: + await send_alert(msg="large") + await load_data() + return {"done": True} +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + // Branch, notify step, load step, return = 4 + assert_eq!(dag.nodes.len(), 4); + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Branch { .. })); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. })); +} + +#[test] +fn test_for_loop_workflow() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def process_item(item: str): ... + +@workflow +async def my_etl(items: list): + for item in items: + await process_item(item=item) + return {"done": True} +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + // LoopStart, step, LoopEnd, return = 4 + assert_eq!(dag.nodes.len(), 4); + assert!(matches!( + dag.nodes[0].node_type, + DagNodeType::LoopStart { .. } + )); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[2].node_type, DagNodeType::LoopEnd)); +} + +#[test] +fn test_reject_step_in_try() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def extract_data(): ... + +@workflow +async def my_etl(): + try: + await extract_data() + except Exception: + pass +"#; + + let result = parse_python_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("try/except")); +} + +#[test] +fn test_reject_step_in_while() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def extract_data(): ... + +@workflow +async def my_etl(): + while True: + await extract_data() +"#; + + let result = parse_python_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("while")); +} + +#[test] +fn test_reject_non_async() { + let code = r#" +from wmill import workflow + +@workflow +def my_etl(): + pass +"#; + + let result = parse_python_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("async")); +} + +#[test] +fn test_reject_missing_await() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def extract_data(): ... + +@workflow +async def my_etl(): + extract_data() +"#; + + let result = parse_python_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("awaited")); +} + +#[test] +fn test_no_workflow_function() { + let code = r#" +async def my_func(): + pass +"#; + + let result = parse_python_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("No @workflow")); +} + +#[test] +fn test_task_with_external_path() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task(path="f/external_script") +async def run_external(x: int): ... + +@workflow +async def my_wf(x: int): + result = await run_external(x=x) + return result +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 2); // 1 step + 1 return (bare `return` is not a step node but walk_return creates one) + match &dag.nodes[0].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "run_external"); + assert_eq!(script, "f/external_script"); + } + _ => panic!("expected Step node"), + } +} diff --git a/backend/parsers/windmill-parser-wac/tests/ts_tests.rs b/backend/parsers/windmill-parser-wac/tests/ts_tests.rs new file mode 100644 index 0000000000..949f326b90 --- /dev/null +++ b/backend/parsers/windmill-parser-wac/tests/ts_tests.rs @@ -0,0 +1,245 @@ +use windmill_parser_wac::dag::DagNodeType; +use windmill_parser_wac::typescript::parse_ts_workflow; + +#[test] +fn test_simple_sequential_ts_workflow() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const extract_data = task(async (url: string) => {}); +const load_data = task(async (data: any) => {}); + +export default workflow(async (url: string) => { + const raw = await extract_data(url); + await load_data(raw); + return { status: "done" }; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 3); // 2 steps + 1 return + assert_eq!(dag.edges.len(), 2); + + // Check params (url — no ctx to skip) + assert_eq!(dag.params.len(), 1); + assert_eq!(dag.params[0].name, "url"); + assert_eq!(dag.params[0].typ.as_deref(), Some("string")); + + match &dag.nodes[0].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "extract_data"); + assert_eq!(script, "extract_data"); + } + _ => panic!("expected Step node"), + } + + match &dag.nodes[1].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "load_data"); + assert_eq!(script, "load_data"); + } + _ => panic!("expected Step node"), + } + + assert!(matches!(dag.nodes[2].node_type, DagNodeType::Return)); + assert!(!dag.source_hash.is_empty()); +} + +#[test] +fn test_parallel_ts_workflow() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const extract_data = task(async (url: string) => {}); +const clean_data = task(async (data: any) => {}); +const compute_stats = task(async (data: any) => {}); +const load_to_warehouse = task(async (rows: any) => {}); + +export default workflow(async (url: string) => { + const raw = await extract_data(url); + + const [cleaned, stats] = await Promise.all([ + clean_data(raw), + compute_stats(raw), + ]); + + await load_to_warehouse(cleaned); + return { status: "done", rows: stats.rowCount }; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + // extract, ParallelStart, clean, stats, ParallelEnd, load, return = 7 + assert_eq!(dag.nodes.len(), 7); + + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::ParallelStart)); + assert!(matches!(dag.nodes[2].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[3].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[4].node_type, DagNodeType::ParallelEnd)); + assert!(matches!(dag.nodes[5].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[6].node_type, DagNodeType::Return)); +} + +#[test] +fn test_conditional_ts_workflow() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const send_alert = task(async (msg: string) => {}); +const load_data = task(async () => {}); + +export default workflow(async (count: number) => { + if (count > 100) { + await send_alert("large"); + } + await load_data(); + return { done: true }; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + // Branch, notify, load, return = 4 + assert_eq!(dag.nodes.len(), 4); + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Branch { .. })); +} + +#[test] +fn test_for_of_ts_workflow() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const process_item = task(async (item: string) => {}); + +export default workflow(async (items: string[]) => { + for (const item of items) { + await process_item(item); + } + return { done: true }; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + // LoopStart, step, LoopEnd, return = 4 + assert_eq!(dag.nodes.len(), 4); + assert!(matches!( + dag.nodes[0].node_type, + DagNodeType::LoopStart { .. } + )); +} + +#[test] +fn test_reject_step_in_try_catch() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const extract_data = task(async () => {}); + +export default workflow(async () => { + try { + await extract_data(); + } catch (e) { + console.log(e); + } +}); +"#; + + let result = parse_ts_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("catch")); +} + +#[test] +fn test_reject_step_in_while_ts() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const extract_data = task(async () => {}); + +export default workflow(async () => { + while (true) { + await extract_data(); + } +}); +"#; + + let result = parse_ts_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("while")); +} + +#[test] +fn test_reject_missing_await_ts() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const extract_data = task(async () => {}); + +export default workflow(async () => { + extract_data(); +}); +"#; + + let result = parse_ts_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("awaited")); +} + +#[test] +fn test_no_workflow_wrapper() { + let code = r#" +export default async function main(ctx: any) { + return {}; +} +"#; + + let result = parse_ts_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("No workflow()")); +} + +#[test] +fn test_variable_declaration_with_step() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const compute = task(async () => {}); + +export default workflow(async () => { + const result = await compute(); + return result; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 2); // step + return + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. })); +} + +#[test] +fn test_task_with_external_path() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const run_external = task("f/external_script", async (x: number) => {}); + +export default workflow(async (x: number) => { + const result = await run_external(x); + return result; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 2); // step + return + match &dag.nodes[0].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "run_external"); + assert_eq!(script, "f/external_script"); + } + _ => panic!("expected Step node"), + } +} diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 1a0d425704..c4891c091b 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -38,6 +38,7 @@ csharp-parser = [ "dep:windmill-parser-csharp"] nu-parser = [ "dep:windmill-parser-nu"] java-parser = [ "dep:windmill-parser-java"] ruby-parser = [ "dep:windmill-parser-ruby"] +wac-parser = [ "dep:windmill-parser-wac"] [dependencies] anyhow.workspace = true @@ -55,6 +56,7 @@ windmill-parser-csharp = { workspace = true, optional = true } windmill-parser-nu = { workspace = true, optional = true } windmill-parser-java = { workspace = true, optional = true } windmill-parser-ruby = { workspace = true, optional = true } +windmill-parser-wac = { workspace = true, optional = true } wasm-bindgen.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-wasm/build.nu b/backend/parsers/windmill-parser-wasm/build.nu index d954366113..cbed629c58 100755 --- a/backend/parsers/windmill-parser-wasm/build.nu +++ b/backend/parsers/windmill-parser-wasm/build.nu @@ -56,6 +56,12 @@ const targets = [ features: "ruby-parser", env: "tree-sitter", }, + { + ident: "wac", + desc: "Workflow-as-Code", + features: "wac-parser", + env: "default", + }, # ^^^ Add new entry here ^^^ ]; # NOTE: This is legacy command for building all, but it is not more used diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index 61cc11d81d..634f2348e1 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -223,4 +223,11 @@ pub fn parse_assets_ansible(code: &str) -> String { } } +#[cfg(feature = "wac-parser")] +#[wasm_bindgen] +pub fn parse_workflow_as_code(code: &str, language: &str) -> String { + let result = windmill_parser_wac::parse_workflow(code, language); + serde_json::to_string(&result).unwrap_or_else(|_| "{\"type\": \"error\"}".to_string()) +} + // for related places search: ADD_NEW_LANG diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 2463995606..44388690e4 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -18,6 +18,7 @@ pub enum AssetKind { Resource, Ducklake, DataTable, + Volume, } #[derive(Serialize, Debug, PartialEq, Clone)] @@ -148,4 +149,5 @@ pub const ASSET_KINDS: &[(&str, AssetKind)] = &[ ("$res:", AssetKind::Resource), ("ducklake://", AssetKind::Ducklake), ("datatable://", AssetKind::DataTable), + ("volume://", AssetKind::Volume), ]; diff --git a/backend/src/main.rs b/backend/src/main.rs index 97980484a5..90acfd76cb 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -38,11 +38,11 @@ use windmill_common::{ agent_workers::AgentConfig, global_settings::{ APP_WORKSPACED_ROUTE_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, - CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, - CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, + CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, + DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, + ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, @@ -99,10 +99,10 @@ use crate::monitor::{ load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, monitor_db, reload_app_workspaced_route_setting, reload_base_url_setting, reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting, - reload_critical_error_channels_setting, reload_extra_pip_index_url_setting, - reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_job_default_timeout_setting, - reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, - reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting, + reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting, + reload_extra_pip_index_url_setting, reload_hub_api_secret_setting, reload_hub_base_url_setting, + reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, + reload_license_key, reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config, reload_uv_index_strategy_setting, reload_worker_config, MonitorIteration, }; @@ -517,6 +517,51 @@ fn print_help() { println!("- At startup, Windmill logs currently set configuration keys for visibility."); } +async fn resync_custom_instance_user_pwd_if_needed(db: &Pool) { + use windmill_common::utils::get_custom_pg_instance_password; + use windmill_common::{get_database_url, PgDatabase}; + + let user_pwd = match get_custom_pg_instance_password(db).await { + Ok(pwd) => pwd, + Err(_) => { + // Setting doesn't exist yet (fresh install or pre-migration), skip check + return; + } + }; + + let mut pg_creds = match get_database_url().await { + Ok(url) => match PgDatabase::parse_uri(&url.as_str().await) { + Ok(creds) => creds, + Err(e) => { + tracing::warn!("Failed to parse database URL for custom_instance_user check: {e}"); + return; + } + }, + Err(e) => { + tracing::warn!("Failed to get database URL for custom_instance_user check: {e}"); + return; + } + }; + + pg_creds.user = Some("custom_instance_user".to_string()); + pg_creds.password = Some(user_pwd); + + match pg_creds.connect().await { + Ok(_) => { + tracing::info!("custom_instance_user password is in sync"); + } + Err(e) => { + tracing::warn!("custom_instance_user password is out of sync ({e}), refreshing..."); + if let Err(e) = windmill_api_settings::refresh_custom_instance_user_pwd_inner(db).await + { + tracing::error!("Failed to refresh custom_instance_user password: {e}"); + } else { + tracing::info!("Successfully refreshed custom_instance_user password"); + } + } + } +} + async fn windmill_main() -> anyhow::Result<()> { let (killpill_tx, mut killpill_rx) = KillpillSender::new(2); let mut monitor_killpill_rx = killpill_tx.subscribe(); @@ -838,6 +883,11 @@ async fn windmill_main() -> anyhow::Result<()> { // NOTE: Variable/resource cache initialization moved to API server in windmill-api + // Check if custom_instance_user password is in sync + if server_mode { + resync_custom_instance_user_pwd_if_needed(&db).await; + } + Connection::Sql(db) }; @@ -1717,6 +1767,11 @@ async fn process_notify_event( tracing::error!(error = %e, "Could not reload critical alert UI setting"); } } + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING => { + if let Err(e) = reload_critical_alerts_on_token_expiry_setting(conn).await { + tracing::error!(error = %e, "Could not reload critical alerts on token expiry setting"); + } + } "workspace_telemetry_enabled" => { // Read the new value from the database and log it let enabled = sqlx::query_scalar!( diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 6baba34a02..37190a5f7a 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -44,19 +44,20 @@ use windmill_common::{ apps::APP_WORKSPACED_ROUTE, auth::create_token_for_owner, ee_oss::CriticalErrorChannel, + email_oss::send_email_if_possible, error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, - CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, - DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, - JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, - KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, - NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING, - OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, - POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, + CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, + DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, + INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, + JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, + MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, + NUGET_CONFIG_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, + POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_INDEX_STRATEGY_SETTING, @@ -76,10 +77,11 @@ use windmill_common::{ DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG, WORKER_GROUP, }, - KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERT_MUTE_UI_ENABLED, - CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, - METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, - OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS, + KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY, + CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, + HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, + MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, + SERVICE_LOG_RETENTION_SECS, }; use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING}; #[cfg(feature = "parquet")] @@ -207,6 +209,10 @@ pub async fn initial_load( tracing::error!("Error loading critical alert mute ui setting: {e:#}"); } + if let Err(e) = reload_critical_alerts_on_token_expiry_setting(conn).await { + tracing::error!("Error loading critical alerts on token expiry setting: {e:#}"); + } + if let Some(db) = conn.as_sql() { if let Err(e) = load_tag_per_workspace_enabled(db).await { tracing::error!("Error loading default tag per workpsace: {e:#}"); @@ -477,6 +483,21 @@ pub async fn reload_critical_alert_mute_ui_setting(conn: &Connection) -> error:: Ok(()) } +pub async fn reload_critical_alerts_on_token_expiry_setting( + conn: &Connection, +) -> error::Result<()> { + if let Ok(Some(serde_json::Value::Bool(t))) = load_value_from_global_settings_with_conn( + conn, + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, + true, + ) + .await + { + CRITICAL_ALERTS_ON_TOKEN_EXPIRY.store(t, Ordering::Relaxed); + } + Ok(()) +} + pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> { let metrics_enabled = load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await; @@ -845,18 +866,82 @@ struct LogFile { hostname: String, } +struct TokenRow { + token_prefix: Option, + label: Option, + email: Option, + workspace_id: Option, +} + +fn is_user_token(label: Option<&str>) -> bool { + match label { + None => true, + Some(l) => l != "session" && !l.starts_with("ephemeral") && !l.starts_with("Ephemeral"), + } +} + +async fn report_token_expiration(db: &DB, token: &TokenRow, expired: bool) { + if !is_user_token(token.label.as_deref()) { + return; + } + let prefix = token.token_prefix.as_deref().unwrap_or("??????????"); + let email_addr = token.email.as_deref().unwrap_or("unknown"); + let token_desc = match token.label.as_deref() { + Some(l) if !l.is_empty() => format!("'{l}' ({prefix}****)"), + _ => format!("{prefix}****"), + }; + + let (alert_message, email_subject, email_body) = if expired { + ( + format!( + "API token {token_desc} of '{email_addr}' has expired and been deleted" + ), + "Windmill: Your API token has expired", + format!( + "Your API token {token_desc} has expired and been deleted.\n\nPlease create a new token if you still need API access." + ), + ) + } else { + ( + format!("API token {token_desc} of '{email_addr}' is expiring soon"), + "Windmill: Your API token is expiring soon", + format!( + "Your API token {token_desc} is expiring soon.\n\nPlease rotate or renew your token to avoid service disruption." + ), + ) + }; + + tracing::info!("{}", alert_message); + if CRITICAL_ALERTS_ON_TOKEN_EXPIRY.load(Ordering::Relaxed) { + report_critical_error( + alert_message, + db.clone(), + token.workspace_id.as_deref(), + None, + ) + .await; + } + if let Some(email) = &token.email { + send_email_if_possible(email_subject, &email_body, email); + } +} + pub async fn delete_expired_items(db: &DB) -> () { - let tokens_deleted_r: std::result::Result, _> = sqlx::query_scalar( + let expired_tokens_r = sqlx::query_as!( + TokenRow, "DELETE FROM token WHERE expiration <= now() - RETURNING concat(substring(token for 10), '*****')", + RETURNING substring(token for 10) as token_prefix, label, email, workspace_id", ) .fetch_all(db) .await; - match tokens_deleted_r { + match expired_tokens_r { Ok(tokens) => { - if tokens.len() > 0 { - tracing::info!("deleted {} tokens: {:?}", tokens.len(), tokens) + if !tokens.is_empty() { + tracing::info!("deleted {} expired tokens", tokens.len()); + for t in &tokens { + report_token_expiration(db, t, true).await; + } } } Err(e) => tracing::error!("Error deleting token: {}", e.to_string()), @@ -1065,6 +1150,41 @@ pub async fn delete_expired_items(db: &DB) -> () { } } +pub async fn check_expiring_tokens(db: &DB) { + // Find tokens expiring within 7 days that still have a pending notification row + let expiring_tokens_r = sqlx::query_as!( + TokenRow, + "DELETE FROM token_expiry_notification n + USING token t + WHERE n.token = t.token + AND n.expiration > now() + AND n.expiration <= now() + interval '7 days' + RETURNING substring(t.token for 10) as token_prefix, t.label, t.email, t.workspace_id", + ) + .fetch_all(db) + .await; + + match expiring_tokens_r { + Ok(tokens) => { + for t in &tokens { + report_token_expiration(db, t, false).await; + } + if !tokens.is_empty() { + tracing::info!("Sent expiration warnings for {} token(s)", tokens.len()); + } + } + Err(e) => tracing::error!("Error checking expiring tokens: {}", e), + } + + // Clean up notification rows whose expiration has passed + if let Err(e) = sqlx::query!("DELETE FROM token_expiry_notification WHERE expiration <= now()") + .execute(db) + .await + { + tracing::error!("Error cleaning up expired token notifications: {}", e); + } +} + /// Delete a batch of expired jobs with LIMIT and SKIP LOCKED for high-scale environments. /// Uses a single transaction per batch to minimize lock duration. /// Returns the number of jobs deleted in this batch. @@ -2052,6 +2172,16 @@ pub async fn monitor_db( } }; + // Run every hour (10 iterations * 30s = 5 minutes) + // Check for tokens expiring within 7 days and send alerts + let check_expiring_tokens_f = async { + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(10) { + if let Some(db) = conn.as_sql() { + check_expiring_tokens(&db).await; + } + } + }; + join!( expired_items_f, zombie_jobs_f, @@ -2073,6 +2203,7 @@ pub async fn monitor_db( cleanup_worker_group_stats_f, native_triggers_sync_f, cleanup_notify_events_f, + check_expiring_tokens_f, ); } diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 01db7590f5..3ece1ce7ef 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -151,6 +151,8 @@ sqs_trigger: path(char), queue_url(char), aws_resource_path(char), message_attri FK: (workspace_id) -> workspace(id) token: token(char), label(char), expiration(ts), workspace_id(char), owner(char), email(char), super_admin(bool), created_at(ts), last_used_at(ts), scopes(text[]), job(uuid) FK: (workspace_id) -> workspace(id) +token_expiry_notification: token(char), expiration(ts) + INDEX: idx_token_expiry_notification_expiration (expiration) tutorial_progress: email(char), progress(bit64), skipped_all(bool) unique_ext_jwt_token: jwt_hash(bigint), last_used_at(ts) usage: id(char), is_workspace(bool), month_(int), usage(int) diff --git a/backend/test_wac_e2e.sh b/backend/test_wac_e2e.sh new file mode 100755 index 0000000000..4379ffde42 --- /dev/null +++ b/backend/test_wac_e2e.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# E2E test for WAC v2 workflow-as-code suspend/resume lifecycle +set -euo pipefail + +BASE_URL="${BASE_URL:-http://localhost:8070}" +TOKEN="${WM_TOKEN:-}" +WORKSPACE="dev" +TIMEOUT=60 # seconds + +# Get auth token if not set +if [ -z "$TOKEN" ]; then + TOKEN=$(curl -s "${BASE_URL}/api/auth/login" \ + -H "Content-Type: application/json" \ + -d '{"email":"admin@windmill.dev","password":"changeme"}' | tr -d '"') +fi + +echo "=== WAC v2 E2E Test ===" +echo "Base URL: $BASE_URL" +echo "" + +WAC_CODE='import { task, workflow } from "windmill-client@1.999.19"; + +const double = task(async (x: number): Promise => { + console.log("[double] START at " + new Date().toISOString()); + await new Promise(r => setTimeout(r, 2000)); + console.log("[double] END at " + new Date().toISOString()); + return x * 2; +}); + +const increment = task(async (x: number): Promise => { + console.log("[increment] START at " + new Date().toISOString()); + await new Promise(r => setTimeout(r, 2000)); + console.log("[increment] END at " + new Date().toISOString()); + return x + 1; +}); + +export const main = workflow(async (x: number = 10) => { + const [doubled, incremented] = await Promise.all([ + double(x), + increment(x), + ]); + const final_result = await double(incremented); + return { doubled, incremented, final_result }; +});' + +echo "Step 1: Submitting preview job..." +JOB_ID=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs/run/preview" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOKEN" \ + -d "$(jq -n --arg code "$WAC_CODE" '{ + content: $code, + language: "bun", + args: {"x": 10} + }')" | tr -d '"') + +echo "Job ID: $JOB_ID" + +if [ -z "$JOB_ID" ] || [ "$JOB_ID" = "null" ]; then + echo "FAIL: Could not create job" + exit 1 +fi + +echo "" +echo "Step 2: Polling for completion (timeout: ${TIMEOUT}s)..." + +START=$SECONDS +LAST_STATUS="" +while true; do + ELAPSED=$((SECONDS - START)) + if [ $ELAPSED -gt $TIMEOUT ]; then + echo "FAIL: Timed out after ${TIMEOUT}s" + # Dump job state for debugging + echo "" + echo "=== Debug info ===" + echo "Parent job queue state:" + source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local + psql "$DATABASE_URL" -c "SELECT id, running, suspend, suspend_until, canceled_by FROM v2_job_queue WHERE id = '$JOB_ID'::uuid" 2>/dev/null + echo "Child jobs:" + psql "$DATABASE_URL" -c "SELECT id, running, suspend, created_at FROM v2_job_queue WHERE parent_job = '$JOB_ID'::uuid ORDER BY created_at" 2>/dev/null + echo "Completed children:" + psql "$DATABASE_URL" -c "SELECT id FROM completed_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null + echo "Checkpoint:" + psql "$DATABASE_URL" -c "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = '$JOB_ID'::uuid" 2>/dev/null + echo "Total child count:" + psql "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null + exit 1 + fi + + # Check completed job + RESULT=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/completed/get_result/${JOB_ID}" \ + -H "Authorization: Bearer $TOKEN" 2>/dev/null) + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/completed/get_result/${JOB_ID}" \ + -H "Authorization: Bearer $TOKEN" 2>/dev/null) + + if [ "$HTTP_CODE" = "200" ]; then + echo "Job completed in ${ELAPSED}s!" + echo "" + echo "Step 3: Checking result..." + echo "Result: $RESULT" + + # Validate + DOUBLED=$(echo "$RESULT" | jq -r '.doubled // empty') + INCREMENTED=$(echo "$RESULT" | jq -r '.incremented // empty') + FINAL=$(echo "$RESULT" | jq -r '.final_result // empty') + + PASS=true + if [ "$DOUBLED" != "20" ]; then + echo "FAIL: doubled = $DOUBLED, expected 20" + PASS=false + fi + if [ "$INCREMENTED" != "11" ]; then + echo "FAIL: incremented = $INCREMENTED, expected 11" + PASS=false + fi + if [ "$FINAL" != "22" ]; then + echo "FAIL: final_result = $FINAL, expected 22" + PASS=false + fi + + if $PASS; then + echo "PASS: All values correct!" + # Check no excessive child jobs + source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local 2>/dev/null + CHILD_COUNT=$(psql -tA "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null) + echo "Total child jobs created: $CHILD_COUNT (expected: 3)" + if [ "$CHILD_COUNT" -gt "3" ]; then + echo "WARN: More children than expected ($CHILD_COUNT > 3)" + fi + exit 0 + else + exit 1 + fi + fi + + # Show progress + STATUS=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/get/${JOB_ID}" \ + -H "Authorization: Bearer $TOKEN" 2>/dev/null | jq -r '.type // empty') + if [ "$STATUS" != "$LAST_STATUS" ]; then + echo " [${ELAPSED}s] Status: $STATUS" + LAST_STATUS="$STATUS" + fi + + # Check for runaway child creation + source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local 2>/dev/null + CHILD_COUNT=$(psql -tA "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null) + if [ "$CHILD_COUNT" -gt "10" ]; then + echo "FAIL: Runaway child creation detected! $CHILD_COUNT children (expected 3)" + echo "" + echo "=== Debug info ===" + psql "$DATABASE_URL" -c "SELECT id, running, suspend, suspend_until FROM v2_job_queue WHERE id = '$JOB_ID'::uuid" 2>/dev/null + psql "$DATABASE_URL" -c "SELECT id, running, suspend, created_at FROM v2_job_queue WHERE parent_job = '$JOB_ID'::uuid ORDER BY created_at LIMIT 20" 2>/dev/null + psql "$DATABASE_URL" -c "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = '$JOB_ID'::uuid" 2>/dev/null + exit 1 + fi + + sleep 1 +done diff --git a/backend/tests/agent_workers.rs b/backend/tests/agent_workers.rs index b6414dc24e..4f12d5f42c 100644 --- a/backend/tests/agent_workers.rs +++ b/backend/tests/agent_workers.rs @@ -1,12 +1,12 @@ #![cfg(all(feature = "private", feature = "agent_worker_server"))] -use windmill_test_utils::*; use serde_json::json; use sqlx::{Pool, Postgres}; use windmill_common::{ jobs::{JobPayload, RawCode}, scripts::ScriptLang, }; +use windmill_test_utils::*; fn bun_code(code: &str) -> RawCode { RawCode { @@ -18,8 +18,8 @@ fn bun_code(code: &str) -> RawCode { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - concurrency_settings: - windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), } } @@ -223,7 +223,10 @@ async fn test_agent_worker_token_and_ping(db: Pool) -> anyhow::Result< .fetch_one(&db) .await?; - assert!(worker_count > 0, "worker ping should be recorded in database"); + assert!( + worker_count > 0, + "worker ping should be recorded in database" + ); // MainLoop ping updates the existing record let resp = http_client @@ -265,3 +268,319 @@ async fn test_agent_worker_multiple_jobs_sequential(db: Pool) -> anyho Ok(()) } + +/// Test the volume HTTP proxy endpoints that agent workers use. +/// +/// Exercises the full volume lifecycle via HTTP: +/// 1. Configure workspace S3 storage (FilesystemStorage) +/// 2. Pre-populate a volume with a file +/// 3. POST /begin — acquire lease, get manifest +/// 4. GET /file/* — download existing file +/// 5. PUT /file/* — upload a new file +/// 6. POST /commit — finalize with stats, release lease +/// 7. Verify DB state and storage +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> { + let (client, _port, _server) = init_client_agent_mode(db.clone()).await; + + // 1. Set up filesystem-based object storage in a temp dir + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null + }); + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + // 2. Pre-populate the volume with a file + let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + std::fs::create_dir_all(&vol_dir)?; + std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; + + let base = client.baseurl(); + let http = client.client(); + let vol_base = format!("{base}/w/test-workspace/volumes/test-vol"); + + // 3. POST /begin — acquire lease, get manifest + permissions + let resp = http + .post(format!("{vol_base}/begin")) + .json(&json!({ + "worker_name": "test-worker-1", + "permissioned_as": "u/test-user" + })) + .send() + .await?; + assert!( + resp.status().is_success(), + "begin should succeed, got: {}", + resp.status() + ); + + let begin_body: serde_json::Value = resp.json().await?; + assert!( + begin_body["writable"].as_bool().unwrap(), + "should be writable" + ); + let manifest = begin_body["manifest"].as_object().unwrap(); + assert!( + manifest.contains_key("hello.txt"), + "manifest should contain hello.txt, got: {manifest:?}" + ); + + // 4. GET /file/* — download the existing file + let resp = http + .get(format!("{vol_base}/file/hello.txt")) + .send() + .await?; + assert!( + resp.status().is_success(), + "file download should succeed, got: {}", + resp.status() + ); + let file_bytes = resp.bytes().await?; + assert_eq!( + file_bytes.as_ref(), + b"hello from volume", + "downloaded file content should match" + ); + + // 5. PUT /file/* — upload a new file + let resp = http + .put(format!("{vol_base}/file/output.txt")) + .body(b"written by agent worker".to_vec()) + .send() + .await?; + assert!( + resp.status().is_success(), + "file upload should succeed, got: {}", + resp.status() + ); + + // 6. POST /commit — finalize: report stats, release lease + let resp = http + .post(format!("{vol_base}/commit")) + .json(&json!({ + "worker_name": "test-worker-1", + "deleted_keys": [], + "symlinks": {}, + "file_count": 2, + "size_bytes": 39 + })) + .send() + .await?; + assert!( + resp.status().is_success(), + "commit should succeed, got: {}", + resp.status() + ); + + // 7. Verify volume DB row was updated + let vol_row = sqlx::query!( + "SELECT size_bytes, file_count, leased_by, lease_until + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await?; + + let vol_row = vol_row.expect("volume row should exist"); + assert_eq!(vol_row.file_count, 2, "file_count should be 2"); + assert_eq!(vol_row.size_bytes, 39, "size_bytes should match"); + assert!(vol_row.leased_by.is_none(), "lease should be released"); + assert!( + vol_row.lease_until.is_none() || vol_row.lease_until.unwrap() < chrono::Utc::now(), + "lease_until should be cleared or in the past" + ); + + // 8. Verify the uploaded file was persisted in storage + let output_path = vol_dir.join("output.txt"); + assert!(output_path.exists(), "output.txt should be in storage"); + let output_content = std::fs::read_to_string(&output_path)?; + assert_eq!(output_content, "written by agent worker"); + + Ok(()) +} + +/// Full E2E test: agent worker in HTTP mode runs a Bun script with a volume mount. +/// +/// The worker pulls the job via HTTP, downloads volume files via the server-side +/// volume proxy endpoints, executes the script, and syncs changes back. +#[cfg(all(feature = "parquet", feature = "enterprise"))] +#[sqlx::test(fixtures("base"))] +async fn test_agent_worker_volume_http_worker_e2e(db: Pool) -> anyhow::Result<()> { + let (_client, port, _server) = init_client_agent_mode(db.clone()).await; + + // 1. Set up filesystem-based object storage in a temp dir + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null + }); + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + // 2. Pre-populate the volume with a file + let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + std::fs::create_dir_all(&vol_dir)?; + std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; + + // 3. Push the job, then run worker with HTTP connection (bun tag) + let code = r#"// volume: test-vol /tmp/data +import { readFileSync, writeFileSync, existsSync } from "fs"; + +export function main() { + const content = readFileSync("/tmp/data/hello.txt", "utf-8"); + writeFileSync("/tmp/data/output.txt", "written by agent worker"); + return { + read_content: content, + output_exists: existsSync("/tmp/data/output.txt"), + }; +}"#; + + let uuid = RunJob::from(JobPayload::Code(bun_code(code))) + .push(&db) + .await; + let listener = listen_for_completed_jobs(&db).await; + + let conn = testing_http_connection_with_tags( + port, + vec!["bun".into(), "flow".into(), "dependency".into()], + ) + .await; + + in_test_worker(conn, listener.find(&uuid), port).await; + + let result = completed_job(uuid, &db).await; + + assert!(result.success, "job should succeed: {:?}", result.result); + let json = result.json_result().expect("should have JSON result"); + assert_eq!(json["read_content"], json!("hello from volume")); + assert_eq!(json["output_exists"], json!(true)); + + // 4. Verify volume DB row was updated + let vol_row = sqlx::query!( + "SELECT size_bytes, file_count, leased_by, lease_until + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await?; + + let vol_row = vol_row.expect("volume row should exist"); + assert!( + vol_row.file_count >= 2, + "should have at least 2 files (hello.txt + output.txt), got: {}", + vol_row.file_count + ); + assert!(vol_row.size_bytes > 0, "size_bytes should be > 0"); + assert!(vol_row.leased_by.is_none(), "lease should be released"); + assert!( + vol_row.lease_until.is_none() || vol_row.lease_until.unwrap() < chrono::Utc::now(), + "lease_until should be cleared or in the past" + ); + + // 5. Verify the new file was written back to the storage + let output_path = vol_dir.join("output.txt"); + assert!( + output_path.exists(), + "output.txt should be synced back to storage" + ); + let output_content = std::fs::read_to_string(&output_path)?; + assert_eq!(output_content, "written by agent worker"); + + Ok(()) +} + +/// Test the volume release endpoint (error/cancel path). +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn test_agent_worker_volume_release(db: Pool) -> anyhow::Result<()> { + let (client, _port, _server) = init_client_agent_mode(db.clone()).await; + + // Set up filesystem storage + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null + }); + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + let base = client.baseurl(); + let http = client.client(); + let vol_base = format!("{base}/w/test-workspace/volumes/test-vol"); + + // Begin (acquire lease) + let resp = http + .post(format!("{vol_base}/begin")) + .json(&json!({ + "worker_name": "test-worker-2", + "permissioned_as": "u/test-user" + })) + .send() + .await?; + assert!(resp.status().is_success(), "begin should succeed"); + + // Verify lease is held + let leased = sqlx::query_scalar!( + "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await? + .flatten(); + assert_eq!(leased.as_deref(), Some("test-worker-2")); + + // Release without commit (simulating error path) + let resp = http + .post(format!("{vol_base}/release")) + .json(&json!({ "worker_name": "test-worker-2" })) + .send() + .await?; + assert!(resp.status().is_success(), "release should succeed"); + + // Verify lease is cleared + let leased = sqlx::query_scalar!( + "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await? + .flatten(); + assert!(leased.is_none(), "lease should be released"); + + Ok(()) +} diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index 15a2b3c27c..1617babb81 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -1,5 +1,6 @@ use sqlx::postgres::Postgres; use sqlx::Pool; +use uuid::Uuid; use windmill_common::jobs::{JobPayload, RawCode}; use windmill_common::scripts::ScriptLang; use windmill_test_utils::*; @@ -1448,3 +1449,240 @@ export function main() { return { a, b }; } ); } } + +// ============================================================================ +// Codebase Mode Tests +// ============================================================================ + +/// Create a TAR archive in memory containing a single `main.js` file. +fn create_codebase_tar(main_js_content: &str) -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + let content = main_js_content.as_bytes(); + let mut header = tar::Header::new_gnu(); + header.set_path("main.js").unwrap(); + header.set_size(content.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder.append(&header, content).unwrap(); + builder.into_inner().unwrap() +} + +/// Place a TAR codebase at the expected cache path for the given job ID and hash. +fn place_codebase_in_cache(job_id: &Uuid, tar_bytes: &[u8], is_esm: bool) { + let codebase_id = if is_esm { + format!("{}.esm.tar", job_id) + } else { + format!("{}.tar", job_id) + }; + let bundle_path = format!("script_bundle/test-workspace/{}", codebase_id); + let cache_path = format!( + "{}/{}.tar", + *windmill_common::worker::ROOT_CACHE_NOMOUNT_DIR, + bundle_path, + ); + let parent = std::path::Path::new(&cache_path).parent().unwrap(); + std::fs::create_dir_all(parent).unwrap(); + std::fs::write(&cache_path, tar_bytes).unwrap(); +} + +#[sqlx::test(fixtures("base"))] +async fn test_cjs_codebase_tar(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let main_js = r#" +module.exports.main = function() { + return "cjs codebase ok"; +}; +"#; + let inner_content = r#"export function main() { return "cjs codebase ok"; }"#; + + let job_id = Uuid::new_v4(); + let tar_bytes = create_codebase_tar(main_js); + place_codebase_in_cache(&job_id, &tar_bytes, false); + + let job = JobPayload::Code(RawCode { + hash: Some(-43), // PREVIEW_IS_TAR_CODEBASE_HASH + content: inner_content.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: Default::default(), + debouncing_settings: Default::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = RunJob::from(job) + .job_id(job_id) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("cjs codebase ok")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_esm_codebase_tar(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let main_js = r#" +export function main() { + return "esm codebase ok"; +} +"#; + let inner_content = r#"export function main() { return "esm codebase ok"; }"#; + + let job_id = Uuid::new_v4(); + let tar_bytes = create_codebase_tar(main_js); + place_codebase_in_cache(&job_id, &tar_bytes, true); + + let job = JobPayload::Code(RawCode { + hash: Some(-45), // PREVIEW_IS_TAR_ESM_CODEBASE_HASH + content: inner_content.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: Default::default(), + debouncing_settings: Default::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = RunJob::from(job) + .job_id(job_id) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("esm codebase ok")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_cjs_codebase_tar_nsjail(db: Pool) -> anyhow::Result<()> { + if std::process::Command::new("nsjail") + .arg("--help") + .output() + .is_err() + { + eprintln!("nsjail not found, skipping test"); + return Ok(()); + } + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let main_js = r#" +module.exports.main = function() { + return "cjs nsjail ok"; +}; +"#; + let inner_content = r#"export function main() { return "cjs nsjail ok"; }"#; + + let job_id = Uuid::new_v4(); + let tar_bytes = create_codebase_tar(main_js); + place_codebase_in_cache(&job_id, &tar_bytes, false); + + let job = JobPayload::Code(RawCode { + hash: Some(-43), + content: inner_content.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: Default::default(), + debouncing_settings: Default::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + use std::sync::atomic::Ordering; + windmill_worker::JOB_ISOLATION.store( + windmill_worker::JobIsolationLevel::NsjailSandboxing as u8, + Ordering::Relaxed, + ); + + let result = RunJob::from(job) + .job_id(job_id) + .run_until_complete(&db, false, port) + .await; + + windmill_worker::JOB_ISOLATION.store( + windmill_worker::JobIsolationLevel::Undefined as u8, + Ordering::Relaxed, + ); + + let json = result.json_result().unwrap(); + assert_eq!(json, serde_json::json!("cjs nsjail ok")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_esm_codebase_tar_nsjail(db: Pool) -> anyhow::Result<()> { + if std::process::Command::new("nsjail") + .arg("--help") + .output() + .is_err() + { + eprintln!("nsjail not found, skipping test"); + return Ok(()); + } + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let main_js = r#" +export function main() { + return "esm nsjail ok"; +} +"#; + let inner_content = r#"export function main() { return "esm nsjail ok"; }"#; + + let job_id = Uuid::new_v4(); + let tar_bytes = create_codebase_tar(main_js); + place_codebase_in_cache(&job_id, &tar_bytes, true); + + let job = JobPayload::Code(RawCode { + hash: Some(-45), + content: inner_content.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: Default::default(), + debouncing_settings: Default::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + use std::sync::atomic::Ordering; + windmill_worker::JOB_ISOLATION.store( + windmill_worker::JobIsolationLevel::NsjailSandboxing as u8, + Ordering::Relaxed, + ); + + let result = RunJob::from(job) + .job_id(job_id) + .run_until_complete(&db, false, port) + .await; + + windmill_worker::JOB_ISOLATION.store( + windmill_worker::JobIsolationLevel::Undefined as u8, + Ordering::Relaxed, + ); + + let json = result.json_result().unwrap(); + assert_eq!(json, serde_json::json!("esm nsjail ok")); + Ok(()) +} diff --git a/backend/tests/scripts/test_volume_with_claude.ts b/backend/tests/scripts/test_volume_with_claude.ts new file mode 100644 index 0000000000..b6230a19ee --- /dev/null +++ b/backend/tests/scripts/test_volume_with_claude.ts @@ -0,0 +1,102 @@ +// volume: agent-memory .claude +// sandbox + +import Anthropic from "@anthropic-ai/sdk"; +import * as fs from "fs"; +import * as path from "path"; + +type Anthropic = { + api_key: string; + model?: string; +}; + +export async function main(anthropic_resource: Anthropic) { + const claudeDir = ".claude"; + const results: Record = {}; + + // --- Step 1: Verify volume is mounted at the relative path --- + results["volume_exists"] = fs.existsSync(claudeDir); + if (!results["volume_exists"]) { + fs.mkdirSync(claudeDir, { recursive: true }); + } + + const testFile = path.join(claudeDir, "mount-check.txt"); + fs.writeFileSync(testFile, "volume mount verified"); + results["volume_writable"] = fs.readFileSync(testFile, "utf-8") === "volume mount verified"; + + // --- Step 2: Create memory directory structure --- + const memoryDir = path.join(claudeDir, "memory"); + fs.mkdirSync(memoryDir, { recursive: true }); + + const memoryFile = path.join(memoryDir, "MEMORY.md"); + fs.writeFileSync(memoryFile, "# Agent Memory\n\nThis file persists across runs.\n"); + results["memory_file_created"] = fs.existsSync(memoryFile); + + // --- Step 3: Call Claude to generate structured content --- + const client = new Anthropic({ apiKey: anthropic_resource.api_key }); + const model = anthropic_resource.model ?? "claude-sonnet-4-20250514"; + + const response = await client.messages.create({ + model, + max_tokens: 256, + messages: [ + { + role: "user", + content: + 'Return a JSON object with exactly these keys: "greeting" (a short hello), "timestamp" (current ISO date you estimate), "items" (array of 3 random fruit names). Only return the JSON, no markdown.', + }, + ], + }); + + const assistantText = + response.content[0].type === "text" ? response.content[0].text : ""; + results["claude_responded"] = assistantText.length > 0; + results["claude_model"] = response.model; + results["claude_stop_reason"] = response.stop_reason; + + let parsed: Record = {}; + try { + parsed = JSON.parse(assistantText); + results["claude_valid_json"] = true; + results["claude_has_greeting"] = "greeting" in parsed; + results["claude_has_items"] = + Array.isArray(parsed.items) && parsed.items.length === 3; + } catch { + results["claude_valid_json"] = false; + } + + // --- Step 4: Write Claude's response to volume --- + const responsePath = path.join(claudeDir, "claude-response.json"); + fs.writeFileSync(responsePath, JSON.stringify(parsed, null, 2)); + results["response_written"] = fs.existsSync(responsePath); + + // --- Step 5: Read back and verify --- + const readBack = fs.readFileSync(responsePath, "utf-8"); + const readParsed = JSON.parse(readBack); + results["readback_matches"] = + JSON.stringify(readParsed) === JSON.stringify(parsed); + + // --- Step 6: List all volume contents --- + const volumeContents = fs.readdirSync(claudeDir); + results["volume_files"] = volumeContents; + results["volume_file_count"] = volumeContents.length; + + // --- Step 7: Verify memory file persists --- + const memoryContent = fs.readFileSync(memoryFile, "utf-8"); + results["memory_persisted"] = memoryContent.includes("Agent Memory"); + + // --- Summary --- + const allChecks = [ + results["volume_exists"] || true, + results["volume_writable"], + results["claude_responded"], + results["claude_valid_json"], + results["response_written"], + results["readback_matches"], + results["memory_file_created"], + results["memory_persisted"], + ]; + results["all_passed"] = allChecks.every(Boolean); + + return results; +} diff --git a/backend/tests/volume_tests.rs b/backend/tests/volume_tests.rs new file mode 100644 index 0000000000..9df30c5615 --- /dev/null +++ b/backend/tests/volume_tests.rs @@ -0,0 +1,637 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::jobs::{JobPayload, RawCode}; +use windmill_common::scripts::ScriptLang; +use windmill_test_utils::*; + +#[sqlx::test(fixtures("base"))] +async fn test_volume_insert(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "test-volume", + 1024_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT workspace_id, name, size_bytes, created_by, last_used_at + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-volume" + ) + .fetch_one(&db) + .await?; + + assert_eq!(row.workspace_id, "test-workspace"); + assert_eq!(row.name, "test-volume"); + assert_eq!(row.size_bytes, 1024); + assert_eq!(row.created_by, "test-user"); + assert!(row.last_used_at.is_none()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_upsert_size(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT (workspace_id, name) DO UPDATE + SET size_bytes = $3, last_used_at = now()", + "test-workspace", + "upsert-vol", + 500_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT size_bytes FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "upsert-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.size_bytes, 500); + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT (workspace_id, name) DO UPDATE + SET size_bytes = $3, last_used_at = now()", + "test-workspace", + "upsert-vol", + 2048_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT size_bytes, last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "upsert-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.size_bytes, 2048); + assert!(row.last_used_at.is_some()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_update_last_used(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "used-vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "used-vol" + ) + .fetch_one(&db) + .await?; + assert!(row.last_used_at.is_none()); + + sqlx::query!( + "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "used-vol" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "used-vol" + ) + .fetch_one(&db) + .await?; + assert!(row.last_used_at.is_some()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_update_nonexistent_noop(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let result = sqlx::query!( + "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "nonexistent-vol" + ) + .execute(&db) + .await?; + + assert_eq!(result.rows_affected(), 0); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_list_multiple(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + for i in 0..5 { + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + format!("vol-{}", i), + (i * 100) as i64, + "test-user" + ) + .execute(&db) + .await?; + } + + let rows = sqlx::query!( + "SELECT name, size_bytes FROM volume WHERE workspace_id = $1 ORDER BY name", + "test-workspace" + ) + .fetch_all(&db) + .await?; + + assert_eq!(rows.len(), 5); + assert_eq!(rows[0].name, "vol-0"); + assert_eq!(rows[0].size_bytes, 0); + assert_eq!(rows[4].name, "vol-4"); + assert_eq!(rows[4].size_bytes, 400); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_delete(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "deleteme", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + let count = sqlx::query_scalar!( + "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "deleteme" + ) + .fetch_one(&db) + .await?; + assert_eq!(count, Some(1)); + + sqlx::query!( + "DELETE FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "deleteme" + ) + .execute(&db) + .await?; + + let count = sqlx::query_scalar!( + "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "deleteme" + ) + .fetch_one(&db) + .await?; + assert_eq!(count, Some(0)); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_workspace_fk_constraint(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let result = sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "nonexistent-workspace", + "vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await; + + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("foreign key"), + "Expected foreign key violation, got: {}", + err + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_primary_key_uniqueness(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "unique-vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + let result = sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "unique-vol", + 200_i64, + "another-user" + ) + .execute(&db) + .await; + + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("duplicate key") || err.contains("unique"), + "Expected unique violation, got: {}", + err + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_extra_perms(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // Insert volume with default (empty) extra_perms + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "perms-vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + // Default extra_perms should be empty object + let row = sqlx::query!( + "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "perms-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.extra_perms, serde_json::json!({})); + + // Set extra_perms via jsonb_set (same pattern as granular_acls.rs) + sqlx::query!( + "UPDATE volume SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2::bool), true) + WHERE workspace_id = $3 AND name = $4", + &vec!["u/alice".to_string()], + true, + "test-workspace", + "perms-vol" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "perms-vol" + ) + .fetch_one(&db) + .await?; + let perms = row.extra_perms.as_object().unwrap(); + assert_eq!(perms.get("u/alice").and_then(|v| v.as_bool()), Some(true)); + + // Remove a permission entry + sqlx::query!( + "UPDATE volume SET extra_perms = extra_perms - $1 + WHERE workspace_id = $2 AND name = $3", + "u/alice", + "test-workspace", + "perms-vol" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "perms-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.extra_perms, serde_json::json!({})); + + Ok(()) +} + +#[test] +fn test_parse_volume_annotations_python() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = r#"# sandbox +# volume: training-data /tmp/training +# volume: models /opt/models + +def main(): + pass +"#; + let volumes = parse_volume_annotations(content, "#"); + assert_eq!(volumes.len(), 2); + assert_eq!(volumes[0].name, "training-data"); + assert_eq!(volumes[0].target, "/tmp/training"); + assert_eq!(volumes[1].name, "models"); + assert_eq!(volumes[1].target, "/opt/models"); +} + +#[test] +fn test_parse_volume_annotations_typescript() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = r#"// sandbox +// volume: datasets /tmp/datasets + +export async function main() { + return "hello"; +} +"#; + let volumes = parse_volume_annotations(content, "//"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].name, "datasets"); + assert_eq!(volumes[0].target, "/tmp/datasets"); +} + +#[test] +fn test_parse_volume_annotations_no_prefix_match() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = "def main():\n pass"; + let volumes = parse_volume_annotations(content, "#"); + assert!(volumes.is_empty()); +} + +#[test] +fn test_parse_volume_annotations_empty_script() { + use windmill_worker_volumes::parse_volume_annotations; + + let volumes = parse_volume_annotations("", "#"); + assert!(volumes.is_empty()); +} + +#[test] +fn test_sandbox_annotation_python() { + use windmill_common::worker::PythonAnnotations; + + let content = "# sandbox\n# volume: data /tmp/data\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); +} + +#[test] +fn test_sandbox_annotation_typescript() { + use windmill_common::worker::TypeScriptAnnotations; + + let content = "// sandbox\n// volume: data /tmp/data\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); +} + +#[test] +fn test_volume_comment_prefix_selection() { + use windmill_common::scripts::ScriptLang; + + let get_prefix = |lang: &ScriptLang| -> &str { + match lang { + ScriptLang::Python3 + | ScriptLang::Bash + | ScriptLang::Powershell + | ScriptLang::Ansible + | ScriptLang::Ruby => "#", + ScriptLang::Deno + | ScriptLang::Bun + | ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Go => "//", + _ => "", + } + }; + + assert_eq!(get_prefix(&ScriptLang::Python3), "#"); + assert_eq!(get_prefix(&ScriptLang::Bash), "#"); + assert_eq!(get_prefix(&ScriptLang::Powershell), "#"); + assert_eq!(get_prefix(&ScriptLang::Ansible), "#"); + assert_eq!(get_prefix(&ScriptLang::Ruby), "#"); + assert_eq!(get_prefix(&ScriptLang::Deno), "//"); + assert_eq!(get_prefix(&ScriptLang::Bun), "//"); + assert_eq!(get_prefix(&ScriptLang::Bunnative), "//"); + assert_eq!(get_prefix(&ScriptLang::Nativets), "//"); + assert_eq!(get_prefix(&ScriptLang::Go), "//"); +} + +#[test] +fn test_volume_mount_struct() { + use windmill_worker_volumes::VolumeMount; + + let mount = VolumeMount { name: "test-vol".to_string(), target: "/mnt/data".to_string() }; + assert_eq!(mount.name, "test-vol"); + assert_eq!(mount.target, "/mnt/data"); +} + +#[test] +fn test_parse_volume_relative_path() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = "// volume: agent-memory .claude\nexport function main() {}"; + let volumes = parse_volume_annotations(content, "//"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].name, "agent-memory"); + assert_eq!(volumes[0].target, ".claude"); +} + +#[test] +fn test_parse_volume_relative_nested_path() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = "# volume: data data/models\ndef main():\n pass"; + let volumes = parse_volume_annotations(content, "#"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].name, "data"); + assert_eq!(volumes[0].target, "data/models"); +} + +#[cfg(feature = "private")] +#[test] +fn test_volume_nsjail_mount() { + use std::path::Path; + use windmill_worker_volumes::volume_nsjail_mount; + + let result = volume_nsjail_mount(Path::new("/tmp/volumes/data"), "/mnt/data"); + assert!(result.contains("src: \"/tmp/volumes/data\"")); + assert!(result.contains("dst: \"/mnt/data\"")); + assert!(result.contains("is_bind: true")); + assert!(result.contains("rw: true")); +} + +#[test] +fn test_sync_stats_default() { + use windmill_worker_volumes::SyncStats; + + let stats = SyncStats { new_size_bytes: 0, file_count: 0, uploaded: 0, skipped: 0 }; + assert_eq!(stats.new_size_bytes, 0); + assert_eq!(stats.file_count, 0); + assert_eq!(stats.uploaded, 0); + assert_eq!(stats.skipped, 0); +} + +#[test] +fn test_asset_kind_volume_variant() { + use windmill_types::assets::AssetKind; + + let kind = AssetKind::Volume; + let serialized = serde_json::to_string(&kind).unwrap(); + assert_eq!(serialized, "\"volume\""); + + let deserialized: AssetKind = serde_json::from_str("\"volume\"").unwrap(); + assert!(matches!(deserialized, AssetKind::Volume)); +} + +/// E2E test: run a bun script with volume mount through a SQL-connected worker. +/// Pre-populates the volume in filesystem storage, verifies the script can read +/// files and write new ones, then checks sync-back to storage and DB state. +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn test_volume_sql_worker_e2e(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // 1. Set up filesystem-based object storage in a temp dir + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null, + "volume_storage": "primary" + }); + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + // 2. Pre-populate the volume with a file (workspace-namespaced path) + let vol_dir = storage_dir + .path() + .join("volumes") + .join("test-workspace") + .join("test-vol"); + std::fs::create_dir_all(&vol_dir)?; + std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; + + // 3. Push the job and run with SQL-connected worker + let code = r#"// volume: test-vol /tmp/data + +import { readFileSync, writeFileSync, existsSync } from "fs"; + +export function main() { + const content = readFileSync("/tmp/data/hello.txt", "utf-8"); + writeFileSync("/tmp/data/output.txt", "written by sql worker"); + return { + read_content: content, + output_exists: existsSync("/tmp/data/output.txt"), + }; +}"#; + + let job = JobPayload::Code(RawCode { + hash: None, + content: code.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port).await; + + assert!(result.success, "job should succeed: {:?}", result.result); + let json = result.json_result().expect("should have JSON result"); + assert_eq!(json["read_content"], json!("hello from volume")); + assert_eq!(json["output_exists"], json!(true)); + + // 4. Verify volume DB row was updated + let vol_row = sqlx::query!( + "SELECT size_bytes, file_count, leased_by, lease_until + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await?; + + let vol_row = vol_row.expect("volume row should exist"); + assert!( + vol_row.file_count >= 2, + "should have at least 2 files (hello.txt + output.txt), got: {}", + vol_row.file_count + ); + assert!(vol_row.size_bytes > 0, "size_bytes should be > 0"); + assert!(vol_row.leased_by.is_none(), "lease should be released"); + + // 5. Verify the new file was written back to storage + let output_path = vol_dir.join("output.txt"); + assert!( + output_path.exists(), + "output.txt should be synced back to storage" + ); + let output_content = std::fs::read_to_string(&output_path)?; + assert_eq!(output_content, "written by sql worker"); + + Ok(()) +} diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 9548986a5a..32fada32fb 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -3548,3 +3548,170 @@ async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow: Ok(()) } + +#[cfg(all(feature = "quickjs", feature = "python"))] +#[sqlx::test(fixtures("base"))] +async fn test_stop_after_all_iters_if_bad_expr_parallel_branchall( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let port = 123; + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [ + { + "id": "a", + "value": { + "branches": [ + {"modules": [{ + "id": "b", + "value": { + "input_transforms": { "n": { "type": "javascript", "expr": "flow_input.n" } }, + "type": "rawscript", + "language": "python3", + "content": "def main(n): return n", + }, + }]} + ], + "type": "branchall", + "parallel": true, + }, + "stop_after_all_iters_if": { + "expr": "invalid!!!syntax", + "skip_if_stopped": false, + }, + }, + ], + })) + .unwrap(); + let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; + + let cjob = RunJob::from(job) + .arg("n", json!(42)) + .run_until_complete(&db, false, port) + .await; + + assert!( + !cjob.success, + "flow should fail when stop_after_all_iters_if has bad expression" + ); + + let result = cjob.json_result().unwrap(); + let error_msg = result["error"]["message"].as_str().unwrap_or(""); + assert!( + error_msg.contains("stop_after_all_iters_if"), + "error should mention stop_after_all_iters_if, got: {error_msg}" + ); + + Ok(()) +} + +#[cfg(all(feature = "quickjs", feature = "python"))] +#[sqlx::test(fixtures("base"))] +async fn test_stop_after_all_iters_if_bad_expr_parallel_forloop( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let port = 123; + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [ + { + "id": "a", + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "result.items" }, + "skip_failures": false, + "parallel": true, + "modules": [{ + "value": { + "input_transforms": { + "n": { "type": "javascript", "expr": "flow_input.iter.value" }, + }, + "type": "rawscript", + "language": "python3", + "content": "def main(n): return n", + }, + }], + }, + "stop_after_all_iters_if": { + "expr": "invalid!!!syntax", + "skip_if_stopped": false, + }, + }, + ], + })) + .unwrap(); + let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; + + let cjob = RunJob::from(job) + .arg("items", json!([1, 2, 3])) + .run_until_complete(&db, false, port) + .await; + + assert!( + !cjob.success, + "flow should fail when stop_after_all_iters_if has bad expression" + ); + + let result = cjob.json_result().unwrap(); + let error_msg = result["error"]["message"].as_str().unwrap_or(""); + assert!( + error_msg.contains("stop_after_all_iters_if"), + "error should mention stop_after_all_iters_if, got: {error_msg}" + ); + + Ok(()) +} + +#[cfg(all(feature = "quickjs", feature = "python"))] +#[sqlx::test(fixtures("base"))] +async fn test_results_length_in_input_transform(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Step a returns a list, step b accesses results.a.length via input transform. + // This tests that the handle_full_regex fast path falls through to QuickJS + // when the SQL JSON path operator can't resolve JS properties like .length. + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "id": "a", + "value": { + "type": "rawscript", + "language": "python3", + "content": "def main(): return [10, 20, 30]", + }, + }, + { + "id": "b", + "value": { + "input_transforms": { + "v": { "type": "javascript", "expr": "results.a.length" }, + }, + "type": "rawscript", + "language": "python3", + "content": "def main(v): return v", + }, + }, + ], + })) + .unwrap(); + + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!( + result, + json!(3), + "results.a.length should resolve to 3, not null" + ); + + Ok(()) +} diff --git a/backend/windmill-api-agent-workers/src/lib.rs b/backend/windmill-api-agent-workers/src/lib.rs index d2913a217c..b02a7098b7 100644 --- a/backend/windmill-api-agent-workers/src/lib.rs +++ b/backend/windmill-api-agent-workers/src/lib.rs @@ -51,4 +51,12 @@ impl AgentCache { pub fn new() -> Self { AgentCache {} } + + pub async fn extract_worker_name( + &self, + _token: &str, + _db: &windmill_common::DB, + ) -> Option { + None + } } diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index 5acb696bd4..d5ec56d00e 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -557,6 +557,14 @@ pub async fn create_token_internal( )); } + register_token_expiry_notification( + &mut *tx, + &token, + token_config.label.as_deref(), + token_config.expiration, + ) + .await; + audit_log( &mut *tx, authed, @@ -572,6 +580,31 @@ pub async fn create_token_internal( Ok(token) } +/// Insert a pending expiry notification row for user tokens that have an expiration. +pub async fn register_token_expiry_notification( + tx: &mut sqlx::PgConnection, + token: &str, + label: Option<&str>, + expiration: Option>, +) { + let Some(expiration) = expiration else { return }; + if label == Some("session") + || label.is_some_and(|l| l.starts_with("ephemeral") || l.starts_with("Ephemeral")) + { + return; + } + if let Err(e) = sqlx::query!( + "INSERT INTO token_expiry_notification (token, expiration) VALUES ($1, $2) ON CONFLICT DO NOTHING", + token, + expiration, + ) + .execute(&mut *tx) + .await + { + tracing::error!("Failed to register token expiry notification: {}", e); + } +} + // ------------ Permission helpers ------------ pub fn get_perm_in_extra_perms_for_authed( diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index 048a4120f5..da9267419d 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -24,7 +24,7 @@ use windmill_common::{ utils::{not_found_if_none, StripPath}, }; -const KINDS: [&str; 18] = [ +const KINDS: [&str; 19] = [ "script", "group_", "resource", @@ -43,6 +43,7 @@ const KINDS: [&str; 18] = [ "gcp_trigger", "sqs_trigger", "email_trigger", + "volume", ]; pub fn workspaced_service() -> Router { @@ -77,7 +78,7 @@ async fn add_granular_acl( let mut tx = user_db.begin(&authed).await?; - let identifier = if kind == "group_" || kind == "folder" { + let identifier = if kind == "group_" || kind == "folder" || kind == "volume" { "name" } else { "path" @@ -89,6 +90,22 @@ async fn add_granular_acl( } else if kind == "group_" { crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db) .await?; + } else if kind == "volume" { + let created_by = sqlx::query_scalar!( + "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("volume '{path}' not found")))?; + // created_by is stored with u/ prefix (from job.permissioned_as) + let owner_username = created_by.strip_prefix("u/").unwrap_or(&created_by); + if owner_username != authed.username { + return Err(Error::NotAuthorized( + "Only the volume owner or an admin can modify permissions".to_string(), + )); + } } else { require_owner_of_path(&authed, path)?; } @@ -243,6 +260,22 @@ async fn remove_granular_acl( } else if kind == "group_" { crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db) .await?; + } else if kind == "volume" { + let created_by = sqlx::query_scalar!( + "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("volume '{path}' not found")))?; + // created_by is stored with u/ prefix (from job.permissioned_as) + let owner_username = created_by.strip_prefix("u/").unwrap_or(&created_by); + if owner_username != authed.username { + return Err(Error::NotAuthorized( + "Only the volume owner or an admin can modify permissions".to_string(), + )); + } } else { require_owner_of_path(&authed, path)?; } @@ -250,7 +283,7 @@ async fn remove_granular_acl( let mut tx = user_db.begin(&authed).await?; - let identifier = if kind == "group_" || kind == "folder" { + let identifier = if kind == "group_" || kind == "folder" || kind == "volume" { "name" } else { "path" @@ -380,7 +413,11 @@ async fn get_granular_acls( let mut tx = user_db.begin(&authed).await?; - let identifier = if kind == "group_" { "name" } else { "path" }; + let identifier = if kind == "group_" || kind == "folder" || kind == "volume" { + "name" + } else { + "path" + }; let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!( "SELECT extra_perms from {kind} WHERE {identifier} = $1 AND workspace_id = $2" )) diff --git a/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs new file mode 100644 index 0000000000..c0f0fe0463 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs @@ -0,0 +1,467 @@ +/*! + * Integration tests for workspace dependencies git sync. + * + * These tests verify that creating, archiving, and deleting workspace dependencies + * triggers deployment callback jobs with the correct arguments for git sync. + * + * Run with enterprise features: + * ```bash + * cargo test --test workspace_dependencies_git_sync --features enterprise,private + * ``` + */ + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use std::time::Duration; + +use windmill_test_utils::*; + +/// Row shape for querying deployment callback jobs from v2_job_queue +#[derive(Debug)] +#[allow(dead_code)] +struct DeploymentCallbackJob { + id: uuid::Uuid, + runnable_path: Option, + args: Option, + kind: String, +} + +/// Poll for deployment callback jobs in the queue for a given script path +async fn get_deployment_callback_jobs( + db: &Pool, + script_path: &str, + timeout: Duration, +) -> anyhow::Result> { + let deadline = tokio::time::Instant::now() + timeout; + loop { + let rows = sqlx::query_as!( + DeploymentCallbackJob, + r#" + SELECT j.id, j.runnable_path, j.args, j.kind::text AS "kind!" + FROM v2_job j + JOIN v2_job_queue q ON j.id = q.id + WHERE j.runnable_path = $1 + AND j.kind = 'deploymentcallback' + ORDER BY j.created_at DESC + "#, + script_path, + ) + .fetch_all(db) + .await?; + + if !rows.is_empty() { + return Ok(rows); + } + + if tokio::time::Instant::now() >= deadline { + // Return empty if timeout - caller will handle assertion + return Ok(vec![]); + } + + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +/// Configure git sync for the test workspace with workspace dependencies enabled +async fn setup_git_sync_config(db: &Pool, sync_script_path: &str) -> anyhow::Result<()> { + let git_sync_config = json!({ + "include_type": ["workspacedependencies"], + "include_path": ["**"], + "repositories": [{ + "script_path": sync_script_path, + "git_repo_resource_path": "$res:u/test-user/test_git_repo", + "use_individual_branch": false, + "group_by_folder": false + }] + }); + + sqlx::query!( + "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", + git_sync_config, + "test-workspace" + ) + .execute(db) + .await?; + + Ok(()) +} + +/// Create a git repository resource for testing +async fn create_git_repo_resource(db: &Pool) -> anyhow::Result<()> { + sqlx::query( + r#" + INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by) + VALUES ('test-workspace', 'u/test-user/test_git_repo', $1::jsonb, 'git_repository', '{}'::jsonb, 'test-user') + ON CONFLICT (workspace_id, path) DO NOTHING + "#, + ) + .bind(json!({ + "url": "https://github.com/test/test.git", + "branch": "main", + "token": "test-token" + })) + .execute(db) + .await?; + + Ok(()) +} + +/// Create a dummy sync script for testing (with version >= 28103 for debouncing support) +async fn create_sync_script(db: &Pool, path: &str) -> anyhow::Result { + let hash: i64 = rand::random::().unsigned_abs() as i64; + sqlx::query( + r#" + INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, kind, lock) + VALUES ('test-workspace', $1, $2, 'sync script', '', + 'export function main(items: any[]) { return { synced: items.length }; }', + 'test-user', 'bun', 'script', '') + "#, + ) + .bind(hash) + .bind(path) + .execute(db) + .await?; + Ok(hash) +} + +/// Create a folder for the versioned script path +async fn create_folder(db: &Pool, name: &str) -> anyhow::Result<()> { + sqlx::query( + r#" + INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by) + VALUES ('test-workspace', $1, $1, ARRAY['u/test-user'], '{}'::jsonb, 'test-user') + ON CONFLICT (workspace_id, name) DO NOTHING + "#, + ) + .bind(name) + .execute(db) + .await?; + Ok(()) +} + +// ============================================================================ +// Tests +// ============================================================================ + +/// Test that creating a workspace dependency triggers a git sync deployment callback +/// with the correct path_type and path arguments. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_create_workspace_dependencies_triggers_git_sync( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + // Setup: Create folder, git repo resource, sync script, and configure git sync + create_folder(&db, "28103").await?; + create_git_repo_resource(&db).await?; + let sync_script_path = "f/28103/test_sync_script"; + create_sync_script(&db, sync_script_path).await?; + setup_git_sync_config(&db, sync_script_path).await?; + + // Start API server + let (client, _port, _server) = init_client(db.clone()).await; + + // Create workspace dependency via API + let response = client + .client() + .post(format!( + "{}/w/test-workspace/workspace_dependencies/create", + client.baseurl() + )) + .json(&json!({ + "workspace_id": "test-workspace", + "language": "python3", + "name": "test-deps", + "content": "requests==2.28.0\nnumpy==1.24.0" + })) + .send() + .await?; + + assert!( + response.status().is_success(), + "Failed to create workspace dependency: {:?}", + response.text().await + ); + + // Wait for deployment callback job to be created + tokio::time::sleep(Duration::from_millis(500)).await; + + // Query for deployment callback jobs + let jobs = get_deployment_callback_jobs(&db, sync_script_path, Duration::from_secs(5)).await?; + + assert!( + !jobs.is_empty(), + "Expected at least one deployment callback job to be created" + ); + + // Verify the job arguments + let job = &jobs[0]; + let args = job.args.as_ref().expect("Job should have args"); + + // Check that path_type is "workspace_dependencies" (or check items array) + // The exact structure depends on whether debouncing is enabled + if let Some(items) = args.get("items") { + // Debounced format: items is an array + let items_arr = items.as_array().expect("items should be an array"); + assert!(!items_arr.is_empty(), "items array should not be empty"); + + let item = &items_arr[0]; + assert_eq!( + item.get("path_type").and_then(|v| v.as_str()), + Some("workspace_dependencies"), + "path_type should be 'workspace_dependencies'" + ); + + // Path should be "dependencies/test-deps.requirements.in" or similar + let path = item.get("path").and_then(|v| v.as_str()).unwrap_or(""); + assert!( + path.contains("dependencies") || path.contains("requirements"), + "path should contain dependencies or requirements: got {}", + path + ); + } else if let Some(path_type) = args.get("path_type") { + // Non-debounced format: path_type is a direct field + assert_eq!( + path_type.as_str(), + Some("workspace_dependencies"), + "path_type should be 'workspace_dependencies'" + ); + } else { + panic!( + "Job args should contain either 'items' array or 'path_type' field: {:?}", + args + ); + } + + Ok(()) +} + +/// Test that archiving a workspace dependency triggers a git sync deployment callback +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_archive_workspace_dependencies_triggers_git_sync( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + // Setup + create_folder(&db, "28103").await?; + create_git_repo_resource(&db).await?; + let sync_script_path = "f/28103/test_sync_script_archive"; + create_sync_script(&db, sync_script_path).await?; + setup_git_sync_config(&db, sync_script_path).await?; + + let (client, _port, _server) = init_client(db.clone()).await; + + // First create a workspace dependency + let create_response = client + .client() + .post(format!( + "{}/w/test-workspace/workspace_dependencies/create", + client.baseurl() + )) + .json(&json!({ + "workspace_id": "test-workspace", + "language": "python3", + "name": "archive-test-deps", + "content": "flask==2.0.0" + })) + .send() + .await?; + + assert!(create_response.status().is_success()); + + // Wait a bit for the create job to be processed + tokio::time::sleep(Duration::from_millis(300)).await; + + // Now archive it + let archive_response = client + .client() + .post(format!( + "{}/w/test-workspace/workspace_dependencies/archive/python3?name=archive-test-deps", + client.baseurl() + )) + .send() + .await?; + + assert!( + archive_response.status().is_success(), + "Failed to archive workspace dependency: {:?}", + archive_response.text().await + ); + + // Wait for deployment callback job + tokio::time::sleep(Duration::from_millis(500)).await; + + // Should have at least 2 items total (create + archive) + // Note: With debouncing enabled (5s window), both operations may be combined into + // a single job with 2 items in the "items" array + let jobs = get_deployment_callback_jobs(&db, sync_script_path, Duration::from_secs(5)).await?; + + let total_items: usize = jobs + .iter() + .map(|job| { + job.args + .as_ref() + .and_then(|args| args.get("items")) + .and_then(|items| items.as_array()) + .map(|arr| arr.len()) + .unwrap_or(1) // Non-debounced jobs count as 1 item + }) + .sum(); + + assert!( + total_items >= 2, + "Expected at least 2 total items (create + archive), got {} items across {} jobs", + total_items, + jobs.len() + ); + + Ok(()) +} + +/// Test that workspace dependencies are NOT synced when workspacedependencies type is excluded +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_workspace_dependencies_respects_include_type_filter( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + // Setup with git sync that EXCLUDES workspacedependencies + create_folder(&db, "28103").await?; + create_git_repo_resource(&db).await?; + let sync_script_path = "f/28103/test_sync_script_filter"; + create_sync_script(&db, sync_script_path).await?; + + // Configure git sync to only include scripts (not workspace dependencies) + let git_sync_config = json!({ + "include_type": ["script"], // Note: workspacedependencies is NOT included + "include_path": ["**"], + "repositories": [{ + "script_path": sync_script_path, + "git_repo_resource_path": "$res:u/test-user/test_git_repo", + "use_individual_branch": false, + "group_by_folder": false + }] + }); + + sqlx::query!( + "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", + git_sync_config, + "test-workspace" + ) + .execute(&db) + .await?; + + let (client, _port, _server) = init_client(db.clone()).await; + + // Create workspace dependency + let response = client + .client() + .post(format!( + "{}/w/test-workspace/workspace_dependencies/create", + client.baseurl() + )) + .json(&json!({ + "workspace_id": "test-workspace", + "language": "python3", + "name": "filtered-deps", + "content": "django==4.0.0" + })) + .send() + .await?; + + assert!(response.status().is_success()); + + // Wait a bit + tokio::time::sleep(Duration::from_secs(1)).await; + + // Should NOT have any deployment callback jobs because workspacedependencies is filtered out + let jobs = + get_deployment_callback_jobs(&db, sync_script_path, Duration::from_millis(500)).await?; + + assert!( + jobs.is_empty(), + "Expected NO deployment callback jobs when workspacedependencies is not in include_type, got {}", + jobs.len() + ); + + Ok(()) +} + +/// Test that the commit message is correctly generated for workspace dependencies +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_workspace_dependencies_commit_message(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // Setup + create_folder(&db, "28103").await?; + create_git_repo_resource(&db).await?; + let sync_script_path = "f/28103/test_sync_script_msg"; + create_sync_script(&db, sync_script_path).await?; + setup_git_sync_config(&db, sync_script_path).await?; + + let (client, _port, _server) = init_client(db.clone()).await; + + // Create workspace dependency + let response = client + .client() + .post(format!( + "{}/w/test-workspace/workspace_dependencies/create", + client.baseurl() + )) + .json(&json!({ + "workspace_id": "test-workspace", + "language": "bun", + "name": null, // unnamed/default dependency + "content": "lodash: ^4.17.21" + })) + .send() + .await?; + + assert!(response.status().is_success()); + + tokio::time::sleep(Duration::from_millis(500)).await; + + let jobs = get_deployment_callback_jobs(&db, sync_script_path, Duration::from_secs(5)).await?; + assert!(!jobs.is_empty()); + + let job = &jobs[0]; + let args = job.args.as_ref().expect("Job should have args"); + + // Check commit message format + if let Some(items) = args.get("items") { + let items_arr = items.as_array().expect("items should be an array"); + if !items_arr.is_empty() { + let commit_msg = items_arr[0] + .get("commit_msg") + .and_then(|v| v.as_str()) + .unwrap_or(""); + + assert!( + commit_msg.contains("[WM]"), + "Commit message should contain '[WM]' prefix: {}", + commit_msg + ); + assert!( + commit_msg.to_lowercase().contains("workspace") + || commit_msg.to_lowercase().contains("dependency") + || commit_msg.to_lowercase().contains("deployed"), + "Commit message should mention workspace dependency or deployed: {}", + commit_msg + ); + } + } else if let Some(commit_msg) = args.get("commit_msg").and_then(|v| v.as_str()) { + assert!( + commit_msg.contains("[WM]"), + "Commit message should contain '[WM]' prefix: {}", + commit_msg + ); + } + + Ok(()) +} diff --git a/backend/windmill-api-jobs/src/job_metrics.rs b/backend/windmill-api-jobs/src/job_metrics.rs index aba8a5f25c..ad316150c0 100644 --- a/backend/windmill-api-jobs/src/job_metrics.rs +++ b/backend/windmill-api-jobs/src/job_metrics.rs @@ -79,7 +79,7 @@ async fn get_job_metrics( >, ) -> error::JsonResult { let records = sqlx::query_as::<_, JobStatsRecord>( - "SELECT * FROM job_stats where workspace_id = $1 and job_id = $2", + "SELECT workspace_id, job_id, metric_id, metric_name, metric_kind, scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float, timeseries_start, offsets_cs FROM job_stats WHERE workspace_id = $1 AND job_id = $2", ) .bind(w_id) .bind(job_id) @@ -91,7 +91,7 @@ async fn get_job_metrics( let mut timeseries_metrics: Vec = vec![]; for record in records { - let metric_id = record.metric_id; + let metric_id = record.metric_id.clone(); match record.metric_kind { MetricKind::ScalarInt => { let value = record.scalar_int.unwrap_or_default() as f64; @@ -102,47 +102,43 @@ async fn get_job_metrics( scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value }); } MetricKind::TimeseriesInt => { - if record.timestamps.clone().unwrap_or_default().len() - != record.timeseries_int.clone().unwrap_or_default().len() - { - tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_int) + let timestamps = resolve_timestamps(&record); + let timeseries_int = record.timeseries_int.unwrap_or_default(); + if timestamps.len() != timeseries_int.len() { + tracing::warn!("Timeseries metric {} has an invalid shape. timestamps: {}, measurements: {}", metric_id, timestamps.len(), timeseries_int.len()); } let (timestamps, timeseries_int) = timeseries_sample( from_timestamp, to_timestamp, timeseries_max_datapoints, - record.timestamps.unwrap_or_default(), - record.timeseries_int.unwrap_or_default(), + timestamps, + timeseries_int, ); - let mut values: Vec = vec![]; - for (idx, value) in timeseries_int.iter().enumerate() { - values.push(DataPoint { - timestamp: timestamps[idx], - value: value.to_owned() as f64, - }); - } + let values: Vec = timestamps + .iter() + .zip(timeseries_int.iter()) + .map(|(ts, v)| DataPoint { timestamp: *ts, value: *v as f64 }) + .collect(); timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values }); } MetricKind::TimeseriesFloat => { - if record.timestamps.clone().unwrap_or_default().len() - != record.timeseries_int.clone().unwrap_or_default().len() - { - tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_float) + let timestamps = resolve_timestamps(&record); + let timeseries_float = record.timeseries_float.unwrap_or_default(); + if timestamps.len() != timeseries_float.len() { + tracing::warn!("Timeseries metric {} has an invalid shape. timestamps: {}, measurements: {}", metric_id, timestamps.len(), timeseries_float.len()); } let (timestamps, timeseries_float) = timeseries_sample( from_timestamp, to_timestamp, timeseries_max_datapoints, - record.timestamps.unwrap_or_default(), - record.timeseries_float.unwrap_or_default(), + timestamps, + timeseries_float, ); - let mut values: Vec = vec![]; - for (idx, value) in timeseries_float.iter().enumerate() { - values.push(DataPoint { - timestamp: timestamps[idx], - value: value.to_owned() as f64, - }); - } + let values: Vec = timestamps + .iter() + .zip(timeseries_float.iter()) + .map(|(ts, v)| DataPoint { timestamp: *ts, value: *v as f64 }) + .collect(); timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values }); } }; @@ -152,6 +148,21 @@ async fn get_job_metrics( let response = JobStatsResponse { metrics_metadata, scalar_metrics, timeseries_metrics }; Ok(Json(response)) } + +/// Reconstruct full timestamps from `timeseries_start` + `offsets_cs` if available, +/// otherwise fall back to legacy `timestamps` column. +fn resolve_timestamps(record: &JobStatsRecord) -> Vec> { + if let (Some(start), Some(offsets)) = (record.timeseries_start, &record.offsets_cs) { + if !offsets.is_empty() { + return offsets + .iter() + .map(|&cs| start + chrono::Duration::milliseconds(cs as i64 * 10)) + .collect(); + } + } + // Legacy fallback: use the full timestamps column + record.timestamps.clone().unwrap_or_default() +} #[derive(Deserialize)] struct JobProgressSetRequest { percent: i32, diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index bf493f9420..0fdabec16d 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -43,8 +43,8 @@ use windmill_common::{ get_database_url, global_settings::{ APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, - CRITICAL_ALERT_MUTE_UI_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, - ENV_SETTINGS, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, + EMAIL_DOMAIN_SETTING, ENV_SETTINGS, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, }, instance_config::{self, ApplyMode, InstanceConfig}, server::Smtp, @@ -316,7 +316,11 @@ pub async fn set_global_setting_internal( ) .execute(db) .await?; - tracing::info!("Set global setting {} to {}", key, v); + tracing::info!( + "Set global setting {} to {}", + key, + instance_config::format_setting_value(&key, &v) + ); } }; @@ -519,6 +523,7 @@ pub async fn get_global_setting( && key != DEFAULT_TAGS_WORKSPACES_SETTING && key != HUB_BASE_URL_SETTING && key != HUB_ACCESSIBLE_URL_SETTING + && key != DISABLE_HUB_SETTING && key != EMAIL_DOMAIN_SETTING && key != APP_WORKSPACED_ROUTE_SETTING { @@ -796,11 +801,7 @@ async fn list_custom_instance_pg_databases( return Ok(Json(result)); } -async fn refresh_custom_instance_user_pwd( - authed: ApiAuthed, - Extension(db): Extension, -) -> JsonResult<()> { - require_super_admin(&db, &authed.email).await?; +pub async fn refresh_custom_instance_user_pwd_inner(db: &DB) -> Result<()> { // 20251208123907_safety_custom_instance_db_user_pwd.up let query = r#" DO $$ @@ -808,7 +809,7 @@ async fn refresh_custom_instance_user_pwd( pwd text; BEGIN SELECT gen_random_uuid()::text INTO pwd; - + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_user') THEN EXECUTE format('ALTER USER custom_instance_user WITH PASSWORD %L', pwd); RAISE NOTICE 'Updated password for existing user custom_instance_user'; @@ -833,7 +834,16 @@ async fn refresh_custom_instance_user_pwd( END $$; "#; - sqlx::query(query).execute(&db).await?; + sqlx::query(query).execute(db).await?; + Ok(()) +} + +async fn refresh_custom_instance_user_pwd( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult<()> { + require_super_admin(&db, &authed.email).await?; + refresh_custom_instance_user_pwd_inner(&db).await?; Ok(Json(())) } diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 17ad6cd7fb..ecd1c5fbd3 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -1850,6 +1850,14 @@ async fn impersonate( .execute(&mut *tx) .await?; + windmill_api_auth::register_token_expiry_notification( + &mut *tx, + &token, + new_token.label.as_deref(), + new_token.expiration, + ) + .await; + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index b698426d53..a03bb3a490 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -29,6 +29,7 @@ windmill-dep-map.workspace = true axum.workspace = true chrono.workspace = true hex.workspace = true +magic-crypt.workspace = true http.workspace = true hyper.workspace = true lazy_static.workspace = true diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 8f3311978a..a85383e496 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -31,7 +31,9 @@ use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::db::UserDB; use windmill_common::users::username_to_permissioned_as; -use windmill_common::variables::{build_crypt, decrypt, encrypt, WORKSPACE_CRYPT_CACHE}; +use windmill_common::variables::{ + build_crypt, decrypt, encrypt, SECRET_SALT, WORKSPACE_CRYPT_CACHE, +}; use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; #[cfg(feature = "enterprise")] use windmill_common::workspaces::GitRepositorySettings; @@ -300,6 +302,8 @@ struct LargeFileStorageWithSecondary { large_file_storage: LargeFileStorage, #[serde(default)] secondary_storage: HashMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + volume_storage: Option, } #[derive(Deserialize, Debug)] struct EditLargeFileStorageConfig { @@ -2418,20 +2422,28 @@ async fn set_encryption_key( )); } + // Build the previous cipher before the transaction (reads from cache/pool) let previous_encryption_key = build_crypt(&db, w_id.as_str()).await?; + let mut tx = db.begin().await?; + sqlx::query!( "UPDATE workspace_key SET key = $1 WHERE workspace_id = $2", request.new_key.clone(), w_id ) - .execute(&db) + .execute(&mut *tx) .await?; - WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); - if !request.skip_reencrypt.unwrap_or(false) { - let new_encryption_key = build_crypt(&db, w_id.as_str()).await?; + // Build the new cipher directly from the key string, since the transaction + // hasn't committed yet and build_crypt() would read the old key from the pool. + let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { + format!("{}{}", request.new_key, salt) + } else { + request.new_key.clone() + }; + let new_encryption_key = magic_crypt::new_magic_crypt!(crypt_key, 256); let mut truncated_new_key = request.new_key.clone(); truncated_new_key.truncate(8); @@ -2445,7 +2457,7 @@ async fn set_encryption_key( "SELECT path, value, is_secret FROM variable WHERE workspace_id = $1", w_id ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; for variable in all_variables { @@ -2466,11 +2478,16 @@ async fn set_encryption_key( w_id, variable.path ) - .execute(&db) + .execute(&mut *tx) .await?; } } + tx.commit().await?; + + // Invalidate the cache only after the transaction has committed + WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); + // Trigger git sync for encryption key changes handle_deployment_metadata( &authed.email, diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 7af8137630..443ec00256 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise"] stripe = [] run_inline = ["dep:windmill-worker", "windmill-api-configs/run_inline"] @@ -70,6 +70,7 @@ windmill-git-sync.workspace = true windmill-indexer = { workspace = true, optional = true } windmill-autoscaling = { workspace = true, optional = true } windmill-worker = { workspace = true, optional = true } +windmill-worker-volumes.workspace = true windmill-dep-map.workspace = true tokio.workspace = true tokio-stream.workspace = true @@ -173,6 +174,7 @@ aws-sdk-bedrock = { workspace = true, optional = true } aws-sdk-bedrockruntime = { workspace = true, optional = true } aws-smithy-types = { workspace = true, optional = true } async-trait.workspace = true +eventsource-stream.workspace = true windmill-jseval.workspace = true tar.workspace = true flate2.workspace = true diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 163217862e..854514dfd8 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.649.0 + version: 1.653.0 title: Windmill API contact: @@ -15198,6 +15198,7 @@ paths: gcp_trigger, sqs_trigger, email_trigger, + volume, ] responses: "200": @@ -15243,6 +15244,7 @@ paths: gcp_trigger, sqs_trigger, email_trigger, + volume, ] requestBody: description: acl to add @@ -15299,6 +15301,7 @@ paths: gcp_trigger, sqs_trigger, email_trigger, + volume, ] requestBody: description: acl to add @@ -16873,6 +16876,9 @@ paths: lost_lock_ownership: description: Is the current indexer service being replaced type: boolean + max_index_time_window_secs: + description: Maximum time window in seconds for indexing + type: number /srch/index/search/service_logs: get: @@ -17282,7 +17288,90 @@ paths: path: type: string description: The asset path - + + + /w/{workspace}/volumes/list: + get: + summary: List all volumes in the workspace + operationId: listVolumes + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: list of volumes + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Volume" + + /w/{workspace}/volumes/storage: + get: + summary: Get the volume storage name (secondary storage) or null for primary + operationId: getVolumeStorage + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: volume storage name or null + content: + application/json: + schema: + type: string + nullable: true + + /w/{workspace}/volumes/create: + post: + summary: Create a new volume + operationId: createVolume + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - name + properties: + name: + type: string + responses: + "200": + description: volume created + content: + text/plain: + schema: + type: string + + /w/{workspace}/volumes/delete/{name}: + delete: + summary: Delete a volume (admin only) + operationId: deleteVolume + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: name + in: path + required: true + schema: + type: string + responses: + "200": + description: volume deleted + content: + text/plain: + schema: + type: string /mcp/w/{workspace}/list_tools: get: @@ -18370,6 +18459,7 @@ components: - trigger - settings - key + - workspacedependencies AIProviderModel: type: object @@ -18717,7 +18807,6 @@ components: required: - path - summary - - description - content - language @@ -23997,6 +24086,7 @@ components: - resource - ducklake - datatable + - volume Asset: type: object properties: @@ -24005,6 +24095,38 @@ components: kind: $ref: "#/components/schemas/AssetKind" required: [path, kind] + Volume: + type: object + required: + - name + - size_bytes + - file_count + - created_at + - created_by + properties: + name: + type: string + size_bytes: + type: integer + format: int64 + file_count: + type: integer + created_at: + type: string + format: date-time + created_by: + type: string + updated_at: + type: string + format: date-time + nullable: true + last_used_at: + type: string + format: date-time + nullable: true + extra_perms: + type: object + additionalProperties: true ProtectionRuleset: type: object description: A workspace protection rule defining restrictions and bypass permissions diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 730df2a875..92ff49f4a4 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -29,7 +29,7 @@ const AI_TIMEOUT_MAX_SECS: u64 = 86400; // 24 hours const AI_TIMEOUT_DEFAULT_SECS: u64 = 3600; // 1 hour const HTTP_POOL_MAX_IDLE_PER_HOST: usize = 10; const HTTP_POOL_IDLE_TIMEOUT_SECS: u64 = 90; -const KEEPALIVE_INTERVAL_SECS: u64 = 15; +pub(crate) const KEEPALIVE_INTERVAL_SECS: u64 = 15; lazy_static::lazy_static! { /// AI request timeout in seconds. @@ -87,7 +87,7 @@ lazy_static::lazy_static! { } }; - static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() + pub(crate) static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() .timeout(std::time::Duration::from_secs(*AI_TIMEOUT_SECS)) .pool_max_idle_per_host(HTTP_POOL_MAX_IDLE_PER_HOST) .pool_idle_timeout(Some(std::time::Duration::from_secs(HTTP_POOL_IDLE_TIMEOUT_SECS))) @@ -378,12 +378,7 @@ impl AIRequestConfig { let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some(); let is_google_ai = matches!(provider, AIProvider::GoogleAI); - // GoogleAI uses OpenAI-compatible endpoint in the proxy (for the chat), but not for the ai agent - let base_url = if is_google_ai { - format!("{}/openai", base_url) - } else { - base_url.to_string() - }; + let base_url = base_url.to_string(); let base_url = base_url.as_str(); // Build URL based on provider @@ -428,6 +423,9 @@ impl AIRequestConfig { if let Some(api_key) = self.api_key { if is_azure { request = request.header("api-key", api_key.clone()) + } else if is_google_ai { + // Native Gemini API uses x-goog-api-key, not Authorization: Bearer + request = request.header("x-goog-api-key", api_key.clone()) } else { request = request.header("authorization", format!("Bearer {}", api_key.clone())) } @@ -611,7 +609,7 @@ fn is_sse_response(headers: &HeaderMap) -> bool { .unwrap_or(false) } -fn inject_keepalives( +pub(crate) fn inject_keepalives( upstream: S, interval: Duration, ) -> impl futures::Stream> @@ -830,6 +828,36 @@ async fn proxy( ai_path = chat_path; } + // Handle GoogleAI (Gemini) using the native Gemini API + if matches!(provider, AIProvider::GoogleAI) { + let api_key = request_config.api_key.as_deref().unwrap_or(""); + let base_url = request_config.base_url.trim_end_matches('/'); + + let mut tx = db.begin().await?; + audit_log( + &mut *tx, + &authed, + "ai.request", + ActionKind::Execute, + &w_id, + Some(&authed.email), + Some([("ai_config_path", &format!("{:?}", ai_path)[..])].into()), + ) + .await?; + tx.commit().await?; + + return match ai_path.as_str() { + "chat/completions" => { + crate::google::handle_google_ai_chat(&body, api_key, base_url).await + } + "models" => crate::google::handle_google_ai_models(api_key, base_url).await, + _ => Err(Error::BadRequest(format!( + "Unsupported Google AI path: {}", + ai_path + ))), + }; + } + // Handle Bedrock-specific logic when the feature is enabled #[cfg(feature = "bedrock")] { diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 1cfbfa6e19..9846912192 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -993,9 +993,18 @@ macro_rules! process_app_multipart { let mut uploaded_js = false; let mut multipart = $multipart; - while let Some(field) = multipart.next_field().await.unwrap() { - let name = field.name().unwrap().to_string(); - let data = field.bytes().await.unwrap(); + while let Some(field) = multipart + .next_field() + .await + .map_err(|e| Error::BadRequest(format!("failed to read multipart field: {e}")))? + { + let name = field + .name() + .ok_or_else(|| Error::BadRequest("multipart field missing name".to_string()))? + .to_string(); + let data = field.bytes().await.map_err(|e| { + Error::BadRequest(format!("failed to read multipart stream: {e}")) + })?; if name == "app" { let app = serde_json::from_slice(&data).map_err(to_anyhow)?; let (ntx, npath, nid) = $internal_fn( @@ -2149,7 +2158,8 @@ async fn execute_component( (email.as_str(), permissioned_as) }; - let end_user_email = get_end_user_email(&db, opt_authed.as_ref(), tokened.token.as_deref()).await; + let end_user_email = + get_end_user_email(&db, opt_authed.as_ref(), tokened.token.as_deref()).await; let (uuid, mut tx) = push( &db, diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 8b0fb44dfe..c8ed841e19 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -284,6 +284,9 @@ pub async fn migrate( 20260207000004, ]; for m in migrator.migrations.iter() { + if m.migration_type.is_down_migration() { + continue; + } if potentially_stale.contains(&m.version) { if let Err(err) = sqlx::query("DELETE FROM _sqlx_migrations WHERE version = $1 AND checksum != $2") diff --git a/backend/windmill-api/src/google.rs b/backend/windmill-api/src/google.rs new file mode 100644 index 0000000000..bc32c20ba4 --- /dev/null +++ b/backend/windmill-api/src/google.rs @@ -0,0 +1,306 @@ +//! Google AI (Gemini API) handler for the AI chat proxy. +//! +//! Handles POST `chat/completions` requests using the native Gemini API, +//! converting from/to OpenAI format so the existing frontend parsers continue to work. +//! +//! Used by `windmill-api/src/ai.rs` when the provider is `GoogleAI`. +//! Shared conversion logic lives in `windmill_common::ai_google`. + +use axum::body::Body; +use bytes::Bytes; +use eventsource_stream::Eventsource; +use futures::StreamExt; +use serde::Deserialize; +use serde_json::json; +use windmill_common::{ + ai_google::{ + gemini_event_to_openai_sse_chunks, gemini_response_to_openai, openai_messages_to_gemini, + parse_gemini_response, parse_gemini_sse_event, sanitize_schema_for_google, + GeminiFunctionDeclaration, GeminiGenerationConfig, GeminiTextRequest, GeminiTool, + }, + ai_types::OpenAIMessage, + error::{Error, Result}, +}; + +use crate::ai::{inject_keepalives, HTTP_CLIENT, KEEPALIVE_INTERVAL_SECS}; + +// ============================================================================ +// Request type (OpenAI format received from the frontend) +// ============================================================================ + +#[derive(Deserialize, Debug)] +struct ChatRequest { + model: String, + messages: Vec, + #[serde(default)] + stream: bool, + #[serde(default)] + temperature: Option, + #[serde(default)] + max_tokens: Option, + #[serde(default)] + tools: Option>, +} + +#[derive(Deserialize, Debug)] +struct ChatRequestTool { + function: ChatRequestToolFunction, +} + +#[derive(Deserialize, Debug)] +struct ChatRequestToolFunction { + name: String, + #[serde(default)] + description: Option, + #[serde(default)] + parameters: Option, +} + +// ============================================================================ +// Public handler +// ============================================================================ + +/// Handle a `chat/completions` POST request using the native Gemini API. +/// +/// Converts the incoming OpenAI-format body to a `GeminiTextRequest`, sends it +/// to the appropriate Gemini endpoint, and converts the response back to the +/// OpenAI SSE or JSON format that the frontend expects. +pub async fn handle_google_ai_chat( + body: &Bytes, + api_key: &str, + base_url: &str, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + let request: ChatRequest = serde_json::from_slice(body) + .map_err(|e| Error::BadRequest(format!("Failed to parse request body: {}", e)))?; + + let (contents, system_instruction) = openai_messages_to_gemini(&request.messages); + + let generation_config = + if request.temperature.is_some() || request.max_tokens.is_some() { + Some(GeminiGenerationConfig { + temperature: request.temperature, + max_output_tokens: request.max_tokens, + response_mime_type: None, + response_schema: None, + }) + } else { + None + }; + + let gemini_tools = request.tools.as_ref().map(|tools| { + let declarations: Vec = tools + .iter() + .map(|t| { + let mut params = t.function.parameters.clone().unwrap_or(json!({})); + sanitize_schema_for_google(&mut params); + GeminiFunctionDeclaration { + name: t.function.name.clone(), + description: t.function.description.clone(), + parameters: params, + } + }) + .collect(); + vec![GeminiTool { + function_declarations: Some(declarations), + google_search: None, + }] + }); + + let gemini_request = GeminiTextRequest { + contents, + tools: gemini_tools, + tool_config: None, + system_instruction, + generation_config, + }; + + let request_body = serde_json::to_string(&gemini_request) + .map_err(|e| Error::internal_err(format!("Failed to serialize Gemini request: {}", e)))?; + + let base_url = base_url.trim_end_matches('/'); + + if request.stream { + handle_streaming(&request.model, request_body, api_key, base_url).await + } else { + handle_non_streaming(&request.model, request_body, api_key, base_url).await + } +} + +// ============================================================================ +// Streaming path +// ============================================================================ + +async fn handle_streaming( + model: &str, + request_body: String, + api_key: &str, + base_url: &str, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + let endpoint = format!("{}/models/{}:streamGenerateContent?alt=sse", base_url, model); + + let response = HTTP_CLIENT + .post(&endpoint) + .header("content-type", "application/json") + .header("x-goog-api-key", api_key) + .body(request_body) + .send() + .await + .map_err(|e| { + Error::internal_err(format!("Failed to send request to Gemini API: {}", e)) + })?; + + if let Err(e) = response.error_for_status_ref() { + let status = e.status().map(|s| s.to_string()).unwrap_or_default(); + let body = response.text().await.unwrap_or_default(); + return Err(Error::AIError(format!("{}: {}", status, body))); + } + + let id = format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()); + let model_str = model.to_string(); + + let gemini_sse_stream = response.bytes_stream().eventsource(); + let openai_sse_stream = async_stream::stream! { + tokio::pin!(gemini_sse_stream); + let mut tool_call_index: usize = 0; + while let Some(event) = gemini_sse_stream.next().await { + match event { + Ok(event) => match parse_gemini_sse_event(&event.data) { + Ok(Some(parsed)) => { + for chunk in gemini_event_to_openai_sse_chunks( + &parsed, &id, &model_str, &mut tool_call_index, + ) { + yield Ok::(Bytes::from(chunk)); + } + } + Ok(None) => {} + Err(e) => tracing::error!("Error parsing Gemini SSE event: {}", e), + }, + Err(e) => tracing::error!("Error reading Gemini SSE stream: {}", e), + } + } + yield Ok::(Bytes::from("data: [DONE]\n\n")); + }; + + let mut headers = http::HeaderMap::new(); + headers.insert("content-type", "text/event-stream".parse().unwrap()); + headers.insert("cache-control", "no-cache".parse().unwrap()); + headers.insert("connection", "keep-alive".parse().unwrap()); + + Ok(( + http::StatusCode::OK, + headers, + Body::from_stream(inject_keepalives( + Box::pin(openai_sse_stream), + std::time::Duration::from_secs(KEEPALIVE_INTERVAL_SECS), + )), + )) +} + +// ============================================================================ +// Model listing +// ============================================================================ + +/// List available Gemini models and convert to OpenAI format. +/// +/// Gemini returns `{ models: [{ name: "models/gemini-2.5-flash", displayName, ... }] }`. +/// The frontend expects OpenAI format `{ data: [{ id: "models/gemini-2.5-flash", ... }] }`. +pub async fn handle_google_ai_models( + api_key: &str, + base_url: &str, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + #[derive(Deserialize)] + struct GeminiModel { + name: String, + #[serde(rename = "displayName", default)] + display_name: String, + } + + #[derive(Deserialize)] + struct GeminiModelsResponse { + #[serde(default)] + models: Vec, + } + + let endpoint = format!("{}/models", base_url.trim_end_matches('/')); + let response = HTTP_CLIENT + .get(&endpoint) + .header("x-goog-api-key", api_key) + .send() + .await + .map_err(|e| Error::internal_err(format!("Failed to fetch Gemini models: {}", e)))?; + + if let Err(e) = response.error_for_status_ref() { + let status = e.status().map(|s| s.to_string()).unwrap_or_default(); + let body = response.text().await.unwrap_or_default(); + return Err(Error::AIError(format!("{}: {}", status, body))); + } + + let gemini_resp: GeminiModelsResponse = response.json().await.map_err(|e| { + Error::internal_err(format!("Failed to parse Gemini models response: {}", e)) + })?; + + let data: Vec = gemini_resp + .models + .into_iter() + .map(|m| { + json!({ + "id": m.name, + "object": "model", + "display_name": m.display_name, + }) + }) + .collect(); + + let body_bytes = serde_json::to_vec(&json!({ "data": data })) + .map_err(|e| Error::internal_err(format!("Failed to serialize models: {}", e)))?; + + let mut headers = http::HeaderMap::new(); + headers.insert("content-type", "application/json".parse().unwrap()); + + Ok((http::StatusCode::OK, headers, Body::from(body_bytes))) +} + +// ============================================================================ +// Non-streaming path +// ============================================================================ + +async fn handle_non_streaming( + model: &str, + request_body: String, + api_key: &str, + base_url: &str, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + let endpoint = format!("{}/models/{}:generateContent", base_url, model); + + let response = HTTP_CLIENT + .post(&endpoint) + .header("content-type", "application/json") + .header("x-goog-api-key", api_key) + .body(request_body) + .send() + .await + .map_err(|e| { + Error::internal_err(format!("Failed to send request to Gemini API: {}", e)) + })?; + + if let Err(e) = response.error_for_status_ref() { + let status = e.status().map(|s| s.to_string()).unwrap_or_default(); + let body = response.text().await.unwrap_or_default(); + return Err(Error::AIError(format!("{}: {}", status, body))); + } + + let body = response.bytes().await.map_err(|e| { + Error::internal_err(format!("Failed to read Gemini response body: {}", e)) + })?; + + let parsed = parse_gemini_response(&body)?; + let openai_response = gemini_response_to_openai(&parsed, model); + + let body_bytes = serde_json::to_vec(&openai_response) + .map_err(|e| Error::internal_err(format!("Failed to serialize response: {}", e)))?; + + let mut headers = http::HeaderMap::new(); + headers.insert("content-type", "application/json".parse().unwrap()); + + Ok((http::StatusCode::OK, headers, Body::from(body_bytes))) +} diff --git a/backend/windmill-api/src/health.rs b/backend/windmill-api/src/health.rs index d861f338ef..60bd290f66 100644 --- a/backend/windmill-api/src/health.rs +++ b/backend/windmill-api/src/health.rs @@ -240,11 +240,7 @@ async fn check_database_detailed(db: &DB) -> DatabaseHealth { let check = check_database_with_latency(db).await; let pool = get_pool_stats(db); - DatabaseHealth { - healthy: check.healthy, - latency_ms: check.latency_ms, - pool, - } + DatabaseHealth { healthy: check.healthy, latency_ms: check.latency_ms, pool } } async fn check_worker_count(db: &DB) -> i64 { @@ -295,13 +291,7 @@ async fn check_workers_detailed(db: &DB) -> WorkersHealth { let healthy = active_count > 0; - WorkersHealth { - healthy, - active_count, - worker_groups, - min_version, - versions, - } + WorkersHealth { healthy, active_count, worker_groups, min_version, versions } } async fn check_queue(db: &DB) -> QueueHealth { @@ -333,10 +323,7 @@ fn get_version() -> String { /// Spawn a background task that performs a health check every 10 seconds. /// Updates the cache and prometheus metrics continuously. -pub fn start_health_check_loop( - db: DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) { +pub fn start_health_check_loop(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) { tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(10)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -550,10 +537,7 @@ async fn health_status( } /// Detailed health check - requires DB authentication (always fresh, no caching) -async fn health_detailed( - _authed: ApiAuthed, - Extension(db): Extension, -) -> impl IntoResponse { +async fn health_detailed(_authed: ApiAuthed, Extension(db): Extension) -> impl IntoResponse { let checked_at = Utc::now(); let database = check_database_detailed(&db).await; let readiness = check_readiness(); @@ -564,12 +548,7 @@ async fn health_detailed( status: HealthStatus::Unhealthy, checked_at, version: get_version(), - checks: HealthChecks { - database, - workers: None, - queue: None, - readiness, - }, + checks: HealthChecks { database, workers: None, queue: None, readiness }, }; return (StatusCode::SERVICE_UNAVAILABLE, Json(response)); } @@ -587,12 +566,7 @@ async fn health_detailed( status, checked_at, version: get_version(), - checks: HealthChecks { - database, - workers: Some(workers), - queue: Some(queue), - readiness, - }, + checks: HealthChecks { database, workers: Some(workers), queue: Some(queue), readiness }, }; let status_code = if status == HealthStatus::Unhealthy { diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index 01f5d67bd3..23f20459c3 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -12,15 +12,15 @@ use windmill_types::s3::StorageResourceType; #[cfg(all(feature = "parquet", not(feature = "private")))] use crate::db::{ApiAuthed, OptJobAuthed, DB}; #[cfg(all(feature = "parquet", not(feature = "private")))] -use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOpts, PutResult}; -#[cfg(not(feature = "private"))] -use windmill_object_store::ObjectStoreResource; -#[cfg(all(feature = "parquet", not(feature = "private")))] use std::sync::Arc; +#[cfg(all(feature = "parquet", not(feature = "private")))] +use windmill_common::db::UserDB; #[cfg(not(feature = "private"))] use windmill_common::error; #[cfg(all(feature = "parquet", not(feature = "private")))] -use windmill_common::db::UserDB; +use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOpts, PutResult}; +#[cfg(not(feature = "private"))] +use windmill_object_store::ObjectStoreResource; #[cfg(all(feature = "parquet", not(feature = "private")))] use bytes::Bytes; diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index ac5a9a306b..184602dc87 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -2255,12 +2255,13 @@ async fn resume_suspended_job_internal( let value = value.unwrap_or(serde_json::Value::Null); verify_suspended_secret(&w_id, &db, job_id, resume_id, &approver, secret).await?; - // Get flow info - works for both step-level (job_id is a step) and flow-level (job_id is the flow) - let (flow_info, is_flow_level) = get_flow_info_for_resume(job_id, &db).await?; + // Get flow info - works for step-level, flow-level, and WAC approval + let (flow_info, is_flow_level, is_wac) = get_flow_info_for_resume(job_id, &db).await?; // For step-level resumes, verify user auth and flow status // For flow-level resumes (pre-approvals), the flow might not be at a suspended step yet - if !is_flow_level { + // For WAC approvals, skip flow status checks (there is no flow) + if !is_flow_level && !is_wac { let parent_flow = GetQuery::new() .without_logs() .without_code() @@ -2322,6 +2323,16 @@ async fn resume_suspended_job_internal( ) .execute(&mut *tx) .await?; + } else if is_wac { + // WAC approval: decrement suspend counter directly on the WAC parent job + if flow_info.suspend > 0 { + sqlx::query!( + "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1", + flow_info.id, + ) + .execute(&mut *tx) + .await?; + } } else if is_flow_level { // For flow-level resumes, decrement the suspend counter if the flow is currently suspended // The approval will be matched when the worker checks for resumes (both step-level and flow-level) @@ -2479,10 +2490,15 @@ struct FlowInfo { email: Option, } -/// Get flow info from either a step job (by looking up its parent) or a flow job directly. -/// Returns (FlowInfo, is_flow_level) where is_flow_level indicates if job_id was a flow job. -async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowInfo, bool)> { - // Single query that determines if job_id is a flow or step, and fetches the appropriate flow info +/// Get flow info from either a step job (by looking up its parent), a flow job directly, +/// or a WAC workflow job (self-suspended for approval). +/// Returns (FlowInfo, is_flow_level, is_wac) where: +/// - is_flow_level: job_id was a flow job (pre-approval) +/// - is_wac: job_id is a WAC workflow suspended for approval (target is itself) +async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowInfo, bool, bool)> { + // Single query that determines if job_id is a flow, step, or WAC job, + // and fetches the appropriate suspended job info. + // For WAC jobs (no parent, not a flow), the job itself is the suspended target. let result = sqlx::query!( r#" WITH job_info AS ( @@ -2496,14 +2512,15 @@ async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowI q.suspend AS "suspend!", j.runnable_path AS script_path, j.permissioned_as_email AS email, - (ji.kind IN ('flow', 'flowpreview')) AS "is_flow_level!" + (ji.kind IN ('flow', 'flowpreview')) AS "is_flow_level!", + (ji.kind NOT IN ('flow', 'flowpreview') AND q.id = ji.id) AS "is_wac!" FROM job_info ji JOIN v2_job_queue q ON q.id = CASE WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id - ELSE ji.parent_job + ELSE COALESCE(ji.parent_job, ji.id) END JOIN v2_job j ON j.id = q.id - JOIN v2_job_status s ON s.id = q.id + LEFT JOIN v2_job_status s ON s.id = q.id FOR UPDATE OF q "#, job_id, @@ -2520,7 +2537,7 @@ async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowI email: Some(result.email), }; - Ok((flow_info, result.is_flow_level)) + Ok((flow_info, result.is_flow_level, result.is_wac)) } async fn get_suspended_flow_info<'c>( diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 528e60b02f..a25a431dba 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -64,6 +64,7 @@ use crate::scim_oss::has_scim_token; use windmill_common::error::AppError; mod ai; +mod google; mod apps; pub mod args; mod audit; @@ -170,6 +171,9 @@ pub mod users_ee; mod users_oss; mod utils; mod variables; +#[cfg(feature = "private")] +pub mod volumes_ee; +mod volumes_oss; pub mod webhook_util; mod workspaces; #[cfg(feature = "private")] @@ -248,6 +252,74 @@ type IndexReader = windmill_indexer::completed_runs_oss::IndexReader; #[cfg(feature = "tantivy")] type ServiceLogIndexReader = windmill_indexer::service_logs_oss::ServiceLogIndexReader; +/// Worker name derived from the agent JWT token, used to authenticate volume operations. +/// Defined unconditionally so volume endpoint handlers can reference it regardless of +/// whether agent_worker_server is enabled (the extension is only populated on the agent path). +#[derive(Clone)] +pub struct AgentWorkerName(pub String); + +/// Middleware that injects a synthetic `ApiAuthed` and JWT-derived worker name +/// into request extensions. +/// +/// Used for volume proxy endpoints under the agent_workers path, where the +/// agent JWT auth layer has already validated the request. The volume handlers +/// need `ApiAuthed` to resolve the workspace S3 client, but the agent JWT +/// format is incompatible with the standard auth extractor. +/// +/// The worker name is extracted from the JWT claims rather than trusting +/// self-reported values in request bodies/query params. +#[cfg(feature = "agent_worker_server")] +async fn inject_agent_authed( + request: axum::extract::Request, + next: axum::middleware::Next, +) -> Response { + let mut request = request; + + // Extract worker name from agent JWT via AgentCache + // (OSS returns None; EE decodes the JWT and returns the worker name) + { + let extracted = { + let token = request + .headers() + .get(axum::http::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.strip_prefix("Bearer ").map(|t| t.to_string())); + let cache = request.extensions().get::>().cloned(); + let db = request.extensions().get::().cloned(); + match (token, cache, db) { + (Some(token), Some(cache), Some(db)) => Some((token, cache, db)), + _ => None, + } + }; + + if let Some((token, cache, db)) = extracted { + if let Some(worker_name) = cache.extract_worker_name(&token, &db).await { + request + .extensions_mut() + .insert(AgentWorkerName(worker_name)); + } + } + } + + request + .extensions_mut() + .insert(windmill_api_auth::OptJobAuthed { + authed: ApiAuthed { + email: "agent-worker@windmill.dev".to_string(), + username: "agent-worker".to_string(), + is_admin: true, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + token_prefix: None, + }, + job_id: None, + }); + next.run(request).await +} + pub async fn run_server( db: DB, job_index_reader: Option, @@ -513,6 +585,7 @@ pub async fn run_server( users::workspaced_service().layer(Extension(argon2.clone())), ) .nest("/variables", variables::workspaced_service()) + .nest("/volumes", volumes_oss::workspaced_service()) .nest("/workers", windmill_api_workers::workspaced_service()) .nest("/workspaces", workspaces::workspaced_service()) .nest("/oidc", oidc_oss::workspaced_service()) @@ -626,7 +699,13 @@ pub async fn run_server( .nest("/w/:workspace_id/agent_workers", { #[cfg(feature = "agent_worker_server")] { - agent_workers_router.layer(Extension(agent_cache.clone())) + agent_workers_router + .nest( + "/volumes", + volumes_oss::agent_workspaced_service() + .layer(axum::middleware::from_fn(inject_agent_authed)), + ) + .layer(Extension(agent_cache.clone())) } #[cfg(not(feature = "agent_worker_server"))] { diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index 604db2dc3e..9a323deef6 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -64,9 +64,10 @@ impl McpBackend for WindmillBackend { auth: &ApiAuthed, workspace_id: &str, favorites_only: bool, + path_prefix: Option<&str>, ) -> BackendResult> { let scope_type = if favorites_only { "favorites" } else { "all" }; - get_items::(&self.user_db, auth, workspace_id, scope_type, "script") + get_items::(&self.user_db, auth, workspace_id, scope_type, "script", path_prefix) .await .map_err(|e| ErrorData::internal_error(e.message, None)) } @@ -76,9 +77,10 @@ impl McpBackend for WindmillBackend { auth: &ApiAuthed, workspace_id: &str, favorites_only: bool, + path_prefix: Option<&str>, ) -> BackendResult> { let scope_type = if favorites_only { "favorites" } else { "all" }; - get_items::(&self.user_db, auth, workspace_id, scope_type, "flow") + get_items::(&self.user_db, auth, workspace_id, scope_type, "flow", path_prefix) .await .map_err(|e| ErrorData::internal_error(e.message, None)) } diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs index d168f8fc48..1cfbfd7085 100644 --- a/backend/windmill-api/src/mcp/utils.rs +++ b/backend/windmill-api/src/mcp/utils.rs @@ -136,6 +136,7 @@ pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen workspace_id: &str, scope_type: &str, item_type: &str, + path_prefix: Option<&str>, ) -> Result, ErrorData> { let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); let fields = vec!["o.path", "o.summary", "o.description", "o.schema"]; @@ -153,6 +154,11 @@ pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen sqlb.and_where("(o.no_main_func IS NOT TRUE OR o.no_main_func IS NULL)"); } + if let Some(prefix) = path_prefix { + let escaped = prefix.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_"); + sqlb.and_where("o.path LIKE ? ESCAPE '\\'".bind(&format!("{}%", escaped))); + } + sqlb.order_by( if item_type == "flow" { "o.edited_at" diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index f3ec05348a..9d95ead840 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -369,7 +369,11 @@ async fn route_job( let s3_object = s3_client.get(&path).await; let s3_object = match s3_object { - Err(windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { .. }) if trigger.is_static_website => { + Err( + windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { + .. + }, + ) if trigger.is_static_website => { // fallback to index.html if the file is not found let path = windmill_object_store::object_store_reexports::Path::from(format!( "{}/index.html", diff --git a/backend/windmill-api/src/volumes_oss.rs b/backend/windmill-api/src/volumes_oss.rs new file mode 100644 index 0000000000..26b1c2cd46 --- /dev/null +++ b/backend/windmill-api/src/volumes_oss.rs @@ -0,0 +1,17 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::volumes_ee::*; + +#[cfg(not(feature = "private"))] +use axum::Router; + +#[cfg(not(feature = "private"))] +pub fn workspaced_service() -> Router { + Router::new() +} + +#[cfg(not(feature = "private"))] +#[allow(dead_code)] +pub fn agent_workspaced_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api/src/workspace_dependencies.rs b/backend/windmill-api/src/workspace_dependencies.rs index d887923f5b..e5c9377b01 100644 --- a/backend/windmill-api/src/workspace_dependencies.rs +++ b/backend/windmill-api/src/workspace_dependencies.rs @@ -16,6 +16,7 @@ use windmill_common::{ use windmill_dep_map::workspace_dependencies::{ trigger_dependents_to_recompute_dependencies_in_the_background, NewWorkspaceDependencies, }; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; use crate::db::ApiAuthed; @@ -37,21 +38,36 @@ async fn create( ) -> error::Result<(StatusCode, String)> { tracing::info!(workspace_id = %nwd.workspace_id, name = ?nwd.name, language = ?nwd.language, "create workspace dependencies"); require_admin(authed.is_admin, &authed.username)?; - Ok(( - StatusCode::CREATED, - format!( - "{}", - nwd.create( - ( - authed.email, - username_to_permissioned_as(&authed.username), - authed.username, - ), - db - ) - .await? - ), - )) + + let dep_path = WorkspaceDependencies::to_path(&nwd.name, nwd.language)?; + let w_id = nwd.workspace_id.clone(); + let email = authed.email.clone(); + let username = authed.username.clone(); + + let id = nwd + .create( + ( + authed.email, + username_to_permissioned_as(&authed.username), + authed.username, + ), + db.clone(), + ) + .await?; + + handle_deployment_metadata( + &email, + &username, + &db, + &w_id, + DeployedObject::WorkspaceDependencies { path: dep_path }, + None, + true, + None, + ) + .await?; + + Ok((StatusCode::CREATED, format!("{}", id))) } #[axum::debug_handler] @@ -92,8 +108,21 @@ async fn archive( tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "archive workspace dependencies"); require_admin(authed.is_admin, &authed.username)?; let db = &db; + let dep_path = WorkspaceDependencies::to_path(¶ms.name, language)?; WorkspaceDependencies::archive(params.name.clone(), language, &w_id, db).await?; + handle_deployment_metadata( + &authed.email, + &authed.username, + db, + &w_id, + DeployedObject::WorkspaceDependencies { path: dep_path.clone() }, + None, + true, + None, + ) + .await?; + trigger_dependents_to_recompute_dependencies_in_the_background( params.name.is_none(), w_id, @@ -103,7 +132,7 @@ async fn archive( username_to_permissioned_as(&authed.username), authed.username, ), - WorkspaceDependencies::to_path(¶ms.name, language)?, + dep_path, db.clone(), ) .await; @@ -121,8 +150,21 @@ async fn delete( tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "delete workspace dependencies"); require_admin(authed.is_admin, &authed.username)?; let db = &db; + let dep_path = WorkspaceDependencies::to_path(¶ms.name, language)?; WorkspaceDependencies::delete(params.name.clone(), language, &w_id, db).await?; + handle_deployment_metadata( + &authed.email, + &authed.username, + db, + &w_id, + DeployedObject::WorkspaceDependencies { path: dep_path.clone() }, + None, + true, + None, + ) + .await?; + trigger_dependents_to_recompute_dependencies_in_the_background( params.name.is_none(), w_id, @@ -132,7 +174,7 @@ async fn delete( username_to_permissioned_as(&authed.username), authed.username, ), - WorkspaceDependencies::to_path(¶ms.name, language)?, + dep_path, db.clone(), ) .await; diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 091c465a61..cd32587648 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -282,6 +282,12 @@ struct SimplifiedSettings { color: Option, #[serde(skip_serializing_if = "Option::is_none")] operator_settings: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_team_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_command_script: Option, } // V1 format: Legacy flat format for backward compatibility (matches main branch exactly) @@ -316,6 +322,12 @@ struct SimplifiedSettingsLegacy { color: Option, #[serde(skip_serializing_if = "Option::is_none")] operator_settings: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_team_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_command_script: Option, } // Internal struct for querying database @@ -335,6 +347,9 @@ struct SettingsRow { mute_critical_alerts: Option, color: Option, operator_settings: Option, + slack_team_id: Option, + slack_name: Option, + slack_command_script: Option, } pub(crate) async fn tarball_workspace( @@ -939,7 +954,10 @@ pub(crate) async fn tarball_workspace( workspace.name as name, mute_critical_alerts, color, - operator_settings + operator_settings, + slack_team_id, + slack_name, + slack_command_script FROM workspace_settings LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id WHERE workspace_id = $1"#, @@ -965,6 +983,9 @@ pub(crate) async fn tarball_workspace( mute_critical_alerts: row.mute_critical_alerts, color: row.color.clone(), operator_settings: row.operator_settings.clone(), + slack_team_id: row.slack_team_id.clone(), + slack_name: row.slack_name.clone(), + slack_command_script: row.slack_command_script.clone(), }; serde_json::to_value(settings) .map(|v| serde_json::to_string_pretty(&v).ok()) @@ -1024,6 +1045,9 @@ pub(crate) async fn tarball_workspace( mute_critical_alerts: row.mute_critical_alerts, color: row.color, operator_settings: row.operator_settings, + slack_team_id: row.slack_team_id, + slack_name: row.slack_name, + slack_command_script: row.slack_command_script, }; serde_json::to_value(settings) .map(|v| serde_json::to_string_pretty(&v).ok()) diff --git a/backend/windmill-common/src/ai_google.rs b/backend/windmill-common/src/ai_google.rs new file mode 100644 index 0000000000..ccf34685e5 --- /dev/null +++ b/backend/windmill-common/src/ai_google.rs @@ -0,0 +1,726 @@ +//! Shared Google AI (Gemini API) types and conversion utilities. +//! +//! This module provides: +//! - Gemini request/response types +//! - OpenAI → Gemini message conversion +//! - Gemini SSE event parsing +//! +//! Used by both windmill-api (chat proxy) and windmill-worker (AI agent). + +use serde::{Deserialize, Serialize}; + +use crate::ai_types::{ContentPart, ExtraContent, GoogleExtraContent, OpenAIContent, OpenAIMessage, ToolDef, UrlCitation}; +use crate::error::Error; + +// ============================================================================ +// Request / Content Types +// ============================================================================ + +/// Inline data for binary content (images). +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GeminiInlineData { + #[serde(rename = "mimeType")] + pub mime_type: String, + pub data: String, +} + +/// A part of content — text, inline data, function call, or function response. +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(untagged)] +pub enum GeminiPart { + Text { + text: String, + }, + InlineData { + #[serde(rename = "inlineData")] + inline_data: GeminiInlineData, + }, + FunctionCall { + #[serde(rename = "functionCall")] + function_call: GeminiFunctionCall, + /// Thought signature for Gemini 3+ models — required when replaying function calls. + #[serde(rename = "thoughtSignature", skip_serializing_if = "Option::is_none")] + thought_signature: Option, + }, + FunctionResponse { + #[serde(rename = "functionResponse")] + function_response: GeminiFunctionResponse, + }, +} + +/// A function call from the model. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GeminiFunctionCall { + pub name: String, + pub args: serde_json::Value, +} + +/// A function response sent back to the model. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GeminiFunctionResponse { + pub name: String, + pub response: serde_json::Value, +} + +/// Content message with an optional role and a list of parts. +#[derive(Serialize, Clone, Debug)] +pub struct GeminiContentMessage { + #[serde(skip_serializing_if = "Option::is_none")] + pub role: Option, + pub parts: Vec, +} + +/// Main request body for `generateContent` / `streamGenerateContent`. +#[derive(Serialize)] +pub struct GeminiTextRequest { + pub contents: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub tools: Option>, + #[serde(rename = "toolConfig", skip_serializing_if = "Option::is_none")] + pub tool_config: Option, + #[serde(rename = "systemInstruction", skip_serializing_if = "Option::is_none")] + pub system_instruction: Option, + #[serde(rename = "generationConfig", skip_serializing_if = "Option::is_none")] + pub generation_config: Option, +} + +/// Tool definition — function declarations and/or Google Search grounding. +#[derive(Serialize)] +pub struct GeminiTool { + #[serde(rename = "functionDeclarations", skip_serializing_if = "Option::is_none")] + pub function_declarations: Option>, + #[serde(rename = "googleSearch", skip_serializing_if = "Option::is_none")] + pub google_search: Option, +} + +/// A single function declaration. +/// +/// `parameters` holds a pre-serialized (and, for the worker, pre-sanitized) JSON Schema. +#[derive(Serialize)] +pub struct GeminiFunctionDeclaration { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + pub parameters: serde_json::Value, +} + +/// Tool configuration controlling when and how functions are called. +#[derive(Serialize)] +pub struct GeminiToolConfig { + #[serde(rename = "functionCallingConfig")] + pub function_calling_config: GeminiFunctionCallingConfig, +} + +/// Function calling mode and optional allow-list. +#[derive(Serialize)] +pub struct GeminiFunctionCallingConfig { + pub mode: String, + #[serde(rename = "allowedFunctionNames", skip_serializing_if = "Option::is_none")] + pub allowed_function_names: Option>, +} + +/// Generation parameters (temperature, token limits, structured output). +#[derive(Serialize)] +pub struct GeminiGenerationConfig { + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(rename = "maxOutputTokens", skip_serializing_if = "Option::is_none")] + pub max_output_tokens: Option, + #[serde(rename = "responseMimeType", skip_serializing_if = "Option::is_none")] + pub response_mime_type: Option, + #[serde(rename = "responseSchema", skip_serializing_if = "Option::is_none")] + pub response_schema: Option, +} + +// ============================================================================ +// Image Generation Types +// ============================================================================ + +/// Request body for Imagen / Gemini image generation. +#[derive(Serialize)] +pub struct GeminiImageRequest { + #[serde(skip_serializing_if = "Option::is_none")] + pub contents: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub instances: Option>, +} + +/// Content wrapper used in `generateContent` image requests. +#[derive(Serialize)] +pub struct GeminiImageContent { + pub parts: Vec, +} + +/// Prompt wrapper for Imagen `predict` endpoint. +#[derive(Serialize)] +pub struct GeminiPredictContent { + pub prompt: String, +} + +/// Top-level response from Gemini/Imagen image generation. +#[derive(Deserialize)] +pub struct GeminiImageResponse { + pub candidates: Option>, + pub predictions: Option>, +} + +#[derive(Deserialize)] +pub struct GeminiImageCandidate { + pub content: GeminiImageCandidateContent, +} + +#[derive(Deserialize)] +pub struct GeminiImageCandidateContent { + pub parts: Vec, +} + +#[derive(Deserialize)] +pub struct GeminiImageCandidatePart { + #[serde(rename = "inlineData")] + pub inline_data: Option, +} + +#[derive(Deserialize)] +pub struct GeminiPredictCandidate { + #[serde(rename = "bytesBase64Encoded")] + pub bytes_base64_encoded: String, +} + +// ============================================================================ +// SSE Response Types +// ============================================================================ + +/// One part inside a streaming candidate — text, function call, or thought signature. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEPart { + #[serde(default)] + pub text: Option, + #[serde(rename = "functionCall")] + pub function_call: Option, + /// Thought signature for Gemini 3+ models. + #[serde(rename = "thoughtSignature")] + pub thought_signature: Option, +} + +/// Function call contained in a streaming part. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEFunctionCall { + pub name: String, + pub args: serde_json::Value, +} + +/// Content block inside a streaming candidate. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEContent { + pub parts: Option>, +} + +/// Web source from a Gemini grounding chunk. +#[derive(Deserialize, Debug)] +pub struct GeminiGroundingChunkWeb { + pub uri: String, + #[serde(default)] + pub title: Option, +} + +/// One grounding chunk (search result) from Gemini web search. +#[derive(Deserialize, Debug)] +pub struct GeminiGroundingChunk { + pub web: Option, +} + +/// Grounding metadata attached to a streaming candidate. +#[derive(Deserialize, Debug)] +pub struct GeminiGroundingMetadata { + #[serde(rename = "groundingChunks", default)] + pub grounding_chunks: Vec, + #[serde(rename = "webSearchQueries", default)] + pub web_search_queries: Vec, +} + +/// One candidate inside a streaming Gemini response. +#[derive(Deserialize, Debug)] +pub struct GeminiSSECandidate { + pub content: Option, + #[serde(rename = "finishReason")] + pub finish_reason: Option, + #[serde(rename = "groundingMetadata")] + pub grounding_metadata: Option, +} + +/// Token usage from the `usageMetadata` field of a Gemini SSE event. +#[derive(Deserialize, Debug, Clone)] +pub struct GeminiUsageMetadata { + #[serde(rename = "promptTokenCount", default)] + pub prompt_token_count: Option, + #[serde(rename = "candidatesTokenCount", default)] + pub candidates_token_count: Option, + #[serde(rename = "totalTokenCount", default)] + pub total_token_count: Option, +} + +/// Top-level structure of one Gemini SSE event. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEEvent { + pub candidates: Option>, + #[serde(rename = "usageMetadata")] + pub usage_metadata: Option, +} + +// ============================================================================ +// Parsed Event Result +// ============================================================================ + +/// A single function call extracted from a Gemini SSE event. +#[derive(Debug)] +pub struct GeminiToolCallEvent { + pub name: String, + pub args: serde_json::Value, + pub thought_signature: Option, +} + +impl GeminiToolCallEvent { + /// Convert the thought signature (if present) into an [`ExtraContent`]. + pub fn to_extra_content(&self) -> Option { + self.thought_signature.as_ref().map(|sig| ExtraContent { + google: Some(GoogleExtraContent { thought_signature: Some(sig.clone()) }), + }) + } +} + +/// Structured result of parsing a Gemini response (streaming SSE event or non-streaming body). +#[derive(Debug, Default)] +pub struct GeminiParsedEvent { + pub text: Option, + pub tool_calls: Vec, + pub annotations: Vec, + pub used_websearch: bool, + pub usage: Option, + pub finish_reason: Option, +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +/// Parse a data URL into `(mime_type, base64_data)`. +/// +/// Expected format: `data:;base64,`. +pub fn parse_data_url(url: &str) -> Option<(String, String)> { + let rest = url.strip_prefix("data:")?; + let (header, data) = rest.split_once(',')?; + let media_type = header.strip_suffix(";base64")?; + Some((media_type.to_string(), data.to_string())) +} + +/// Find the function name associated with a `tool_call_id` by scanning prior messages. +pub fn find_gemini_function_name(messages: &[OpenAIMessage], tool_call_id: &str) -> String { + messages + .iter() + .filter_map(|msg| msg.tool_calls.as_ref()) + .flatten() + .find(|tc| tc.id == tool_call_id) + .map(|tc| tc.function.name.clone()) + .unwrap_or_else(|| "unknown_function".to_string()) +} + +/// Convert an [`OpenAIContent`] value to a list of [`GeminiPart`]s. +/// +/// Handles text and `image_url` (data URLs). `S3Object` variants are skipped here; +/// the worker handles them by downloading and injecting inline data beforehand. +pub fn convert_content_to_gemini_parts(content: &OpenAIContent) -> Vec { + match content { + OpenAIContent::Text(text) if !text.is_empty() => { + vec![GeminiPart::Text { text: text.clone() }] + } + OpenAIContent::Text(_) => vec![], + OpenAIContent::Parts(parts) => parts + .iter() + .filter_map(|part| match part { + ContentPart::Text { text } if !text.is_empty() => { + Some(GeminiPart::Text { text: text.clone() }) + } + ContentPart::ImageUrl { image_url } => { + parse_data_url(&image_url.url).map(|(mime_type, data)| { + GeminiPart::InlineData { + inline_data: GeminiInlineData { mime_type, data }, + } + }) + } + // S3Objects are handled by the worker + _ => None, + }) + .collect(), + } +} + +/// Convert OpenAI-format messages to Gemini `contents` and an optional `systemInstruction`. +/// +/// Returns `(contents, system_instruction)`. +/// +/// `S3Object` images in content parts are skipped (the worker pre-converts them). +/// Tool call history is preserved correctly for multi-turn agent conversations. +pub fn openai_messages_to_gemini( + messages: &[OpenAIMessage], +) -> (Vec, Option) { + let mut contents: Vec = Vec::new(); + let mut system_instruction: Option = None; + + for msg in messages { + match msg.role.as_str() { + "system" => { + if let Some(content) = &msg.content { + let parts = convert_content_to_gemini_parts(content); + if !parts.is_empty() { + system_instruction = + Some(GeminiContentMessage { role: None, parts }); + } + } + } + "tool" => { + if let (Some(tool_call_id), Some(content)) = + (&msg.tool_call_id, &msg.content) + { + let func_name = find_gemini_function_name(messages, tool_call_id); + let response_text = match content { + OpenAIContent::Text(text) => text.clone(), + OpenAIContent::Parts(parts) => parts + .iter() + .filter_map(|p| { + if let ContentPart::Text { text } = p { + Some(text.as_str()) + } else { + None + } + }) + .collect::>() + .join(" "), + }; + contents.push(GeminiContentMessage { + role: Some("user".to_string()), + parts: vec![GeminiPart::FunctionResponse { + function_response: GeminiFunctionResponse { + name: func_name, + response: serde_json::json!({ "result": response_text }), + }, + }], + }); + } + } + role => { + let gemini_role = if role == "assistant" { "model" } else { "user" }; + let mut parts: Vec = Vec::new(); + + if let Some(content) = &msg.content { + parts.extend(convert_content_to_gemini_parts(content)); + } + + if let Some(tool_calls) = &msg.tool_calls { + for tc in tool_calls { + let args: serde_json::Value = + serde_json::from_str(&tc.function.arguments).unwrap_or_default(); + let thought_signature = tc + .extra_content + .as_ref() + .and_then(|ec| ec.google.as_ref()) + .and_then(|g| g.thought_signature.clone()); + parts.push(GeminiPart::FunctionCall { + function_call: GeminiFunctionCall { + name: tc.function.name.clone(), + args, + }, + thought_signature, + }); + } + } + + if !parts.is_empty() { + contents.push(GeminiContentMessage { + role: Some(gemini_role.to_string()), + parts, + }); + } + } + } + } + + (contents, system_instruction) +} + +/// Convert OpenAI tool definitions to Gemini format. +/// +/// `tool_params` must be pre-serialized (and, for the worker, pre-sanitized for Google) +/// JSON schema values, one per entry in `tools` in the same order. +pub fn openai_tools_to_gemini( + tools: &[ToolDef], + tool_params: &[serde_json::Value], + has_websearch: bool, +) -> Option> { + let mut gemini_tools: Vec = Vec::new(); + + let declarations: Vec = tools + .iter() + .zip(tool_params.iter()) + .map(|(t, params)| GeminiFunctionDeclaration { + name: t.function.name.clone(), + description: t.function.description.clone(), + parameters: params.clone(), + }) + .collect(); + + if !declarations.is_empty() { + gemini_tools.push(GeminiTool { + function_declarations: Some(declarations), + google_search: None, + }); + } + + if has_websearch { + gemini_tools.push(GeminiTool { + function_declarations: None, + google_search: Some(serde_json::json!({})), + }); + } + + if gemini_tools.is_empty() { + None + } else { + Some(gemini_tools) + } +} + +/// Parse one Gemini SSE data line into a [`GeminiParsedEvent`]. +/// +/// Returns `Ok(None)` for empty data or unrecognised payloads (e.g. `"[DONE]"`). +/// Logs a warning and returns `Ok(None)` on JSON parse errors rather than propagating. +pub fn parse_gemini_sse_event(data: &str) -> Result, Error> { + if data.is_empty() || data == "[DONE]" { + return Ok(None); + } + + let event: GeminiSSEEvent = match serde_json::from_str(data) { + Ok(e) => e, + Err(e) => { + tracing::error!("Failed to parse Gemini SSE event {}: {}", data, e); + return Ok(None); + } + }; + + let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() }; + + let Some(candidates) = event.candidates else { + return Ok(Some(parsed)); + }; + + extract_candidates_into(&candidates, &mut parsed); + + Ok(Some(parsed)) +} + +/// Parse a non-streaming Gemini `generateContent` response body. +pub fn parse_gemini_response(data: &[u8]) -> Result { + let event: GeminiSSEEvent = serde_json::from_slice(data) + .map_err(|e| Error::internal_err(format!("Failed to parse Gemini response: {}", e)))?; + + let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() }; + + if let Some(candidates) = event.candidates { + extract_candidates_into(&candidates, &mut parsed); + } + + Ok(parsed) +} + +// ============================================================================ +// Gemini → OpenAI Format Conversion +// ============================================================================ + +/// Convert a `GeminiParsedEvent` from a non-streaming response to an OpenAI chat completion JSON. +pub fn gemini_response_to_openai(parsed: &GeminiParsedEvent, model: &str) -> serde_json::Value { + let content = parsed.text.as_deref().unwrap_or_default(); + + let tool_calls: Vec = parsed + .tool_calls + .iter() + .enumerate() + .map(|(i, tc)| { + serde_json::json!({ + "index": i, + "id": format!("call_{}", uuid::Uuid::new_v4().simple()), + "type": "function", + "function": { + "name": tc.name, + "arguments": serde_json::to_string(&tc.args).unwrap_or_default() + } + }) + }) + .collect(); + + let finish_reason = parsed + .finish_reason + .as_deref() + .map(|r| r.to_lowercase()) + .unwrap_or_else(|| "stop".to_string()); + + let usage = parsed.usage.as_ref().map(|u| { + serde_json::json!({ + "prompt_tokens": u.prompt_token_count.unwrap_or(0), + "completion_tokens": u.candidates_token_count.unwrap_or(0), + "total_tokens": u.total_token_count.unwrap_or(0), + }) + }); + + let mut message = serde_json::json!({ + "role": "assistant", + "content": content, + }); + if !tool_calls.is_empty() { + message["tool_calls"] = serde_json::json!(tool_calls); + } + + serde_json::json!({ + "id": format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()), + "object": "chat.completion", + "model": model, + "choices": [{ + "index": 0, + "message": message, + "finish_reason": finish_reason, + }], + "usage": usage, + }) +} + +/// Convert a `GeminiParsedEvent` from a streaming SSE event into OpenAI-format SSE lines. +/// +/// Returns the serialized `"data: {...}\n\n"` lines ready to be written to the response stream. +/// `tool_call_index` is mutated to track the running index across multiple SSE events. +pub fn gemini_event_to_openai_sse_chunks( + parsed: &GeminiParsedEvent, + id: &str, + model: &str, + tool_call_index: &mut usize, +) -> Vec { + let mut chunks = Vec::new(); + + if let Some(text) = &parsed.text { + let chunk = serde_json::json!({ + "id": id, + "object": "chat.completion.chunk", + "model": model, + "choices": [{ + "index": 0, + "delta": { "content": text }, + "finish_reason": null, + }] + }); + chunks.push(format!("data: {}\n\n", chunk)); + } + + for tc in &parsed.tool_calls { + let args_str = serde_json::to_string(&tc.args).unwrap_or_default(); + let call_id = format!("call_{}", uuid::Uuid::new_v4().simple()); + let chunk = serde_json::json!({ + "id": id, + "object": "chat.completion.chunk", + "model": model, + "choices": [{ + "index": 0, + "delta": { + "tool_calls": [{ + "index": *tool_call_index, + "id": call_id, + "type": "function", + "function": { + "name": tc.name, + "arguments": args_str, + } + }] + }, + "finish_reason": null, + }] + }); + chunks.push(format!("data: {}\n\n", chunk)); + *tool_call_index += 1; + } + + chunks +} + +/// Recursively remove JSON Schema fields unsupported by the Gemini API. +pub fn sanitize_schema_for_google(value: &mut serde_json::Value) { + const UNSUPPORTED: &[&str] = &[ + "additionalProperties", + "strict", + "$schema", + "default", + "exclusiveMinimum", + "exclusiveMaximum", + "const", + "multipleOf", + ]; + + if let Some(obj) = value.as_object_mut() { + for field in UNSUPPORTED { + obj.remove(*field); + } + for v in obj.values_mut() { + sanitize_schema_for_google(v); + } + } else if let Some(arr) = value.as_array_mut() { + for v in arr.iter_mut() { + sanitize_schema_for_google(v); + } + } +} + +// ============================================================================ +// Internal Helpers +// ============================================================================ + +fn extract_candidates_into(candidates: &[GeminiSSECandidate], parsed: &mut GeminiParsedEvent) { + for candidate in candidates { + if let Some(content) = &candidate.content { + if let Some(parts) = &content.parts { + for part in parts { + if let Some(text) = &part.text { + if !text.is_empty() { + match parsed.text.as_mut() { + Some(existing) => existing.push_str(text), + None => parsed.text = Some(text.clone()), + } + } + } + + if let Some(function_call) = &part.function_call { + parsed.tool_calls.push(GeminiToolCallEvent { + name: function_call.name.clone(), + args: function_call.args.clone(), + thought_signature: part.thought_signature.clone(), + }); + } + } + } + } + + if candidate.finish_reason.is_some() { + parsed.finish_reason = candidate.finish_reason.clone(); + } + + if let Some(grounding) = &candidate.grounding_metadata { + if !grounding.web_search_queries.is_empty() || !grounding.grounding_chunks.is_empty() { + parsed.used_websearch = true; + } + for chunk in &grounding.grounding_chunks { + if let Some(web) = &chunk.web { + parsed.annotations.push(UrlCitation { + start_index: 0, + end_index: 0, + url: web.uri.clone(), + title: web.title.clone(), + }); + } + } + } + } +} diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 669614d64a..2968493ecc 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -72,6 +72,7 @@ pub fn asset_kind_from_parser(parser_kind: windmill_parser::asset_parser::AssetK windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource, windmill_parser::asset_parser::AssetKind::Ducklake => AssetKind::Ducklake, windmill_parser::asset_parser::AssetKind::DataTable => AssetKind::DataTable, + windmill_parser::asset_parser::AssetKind::Volume => AssetKind::Volume, } } diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index ccd9b5d2a0..c0555f5093 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -95,6 +95,51 @@ impl PermsCache { } } +/// Check a user's access level against an `extra_perms` JSONB object. +/// +/// Returns `None` if the user has no matching entry (no access). +/// Returns `Some(true)` if the user (or any of their groups) has write access. +/// Returns `Some(false)` if the user (or any of their groups) has read-only access. +pub fn check_extra_perms( + extra_perms: &serde_json::Map, + username: &str, + groups: &[String], +) -> Option { + // Check direct user permission + let user_key = if username.starts_with("u/") { + username.to_string() + } else { + format!("u/{username}") + }; + if let Some(v) = extra_perms.get(&user_key) { + return Some(v.as_bool().unwrap_or(false)); + } + + // Check group permissions — return highest access level found + let mut found = false; + let mut write = false; + for g in groups { + let key = if g.starts_with("g/") { + g.to_string() + } else { + format!("g/{g}") + }; + if let Some(v) = extra_perms.get(&key) { + found = true; + if v.as_bool().unwrap_or(false) { + write = true; + break; + } + } + } + + if found { + Some(write) + } else { + None + } +} + pub fn has_expired(expiration_time: DateTime, take: Option) -> bool { let now = Utc::now(); diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index eaf7d857c1..49dcf25450 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -78,6 +78,8 @@ pub enum Error { AIError(String), #[error("{0}")] AlreadyCompleted(String), + #[error("WAC job suspended: {0}")] + WacSuspended(String), #[error("Find python error: {0}")] FindPythonError(String), #[error("Problem with arguments: {0}")] @@ -108,6 +110,7 @@ impl Error { Self::JsonErr(_) => "JsonErr", Self::AIError(_) => "AIError", Self::AlreadyCompleted(_) => "AlreadyCompleted", + Self::WacSuspended(_) => "WacSuspended", Self::FindPythonError(_) => "FindPythonError", Self::ArgumentErr(_) => "ArgumentErr", Self::Generic(_, _) => "Generic", diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 3347127303..b2d961e173 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -44,9 +44,11 @@ pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url"; +pub const DISABLE_HUB_SETTING: &str = "disable_hub"; pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels"; pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui"; pub const CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING: &str = "critical_alerts_on_db_oversize"; +pub const CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING: &str = "critical_alerts_on_token_expiry"; pub const DEV_INSTANCE_SETTING: &str = "dev_instance"; pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; diff --git a/backend/windmill-common/src/indexer.rs b/backend/windmill-common/src/indexer.rs index 45a924748d..23bde13795 100644 --- a/backend/windmill-common/src/indexer.rs +++ b/backend/windmill-common/src/indexer.rs @@ -13,6 +13,7 @@ pub struct TantivyIndexerSettings { pub refresh_index_period: u64, pub refresh_log_index_period: u64, pub max_indexed_job_log_size: usize, + pub max_index_time_window_secs: i64, pub should_clear_job_index: bool, pub should_clear_log_index: bool, } @@ -26,6 +27,7 @@ impl Default for TantivyIndexerSettings { refresh_index_period: 300, refresh_log_index_period: 300, max_indexed_job_log_size: 1_000_000, + max_index_time_window_secs: 60 * 60 * 24 * 7, // 7 days should_clear_job_index: false, should_clear_log_index: false, } @@ -39,6 +41,7 @@ pub struct TantivyIndexerSettingsOpt { pub refresh_index_period: Option, pub refresh_log_index_period: Option, pub max_indexed_job_log_size: Option, + pub max_index_time_window_secs: Option, pub should_clear_job_index: Option, pub should_clear_log_index: Option, } @@ -58,6 +61,7 @@ pub async fn load_indexer_config(db: &DB) -> error::Result error::Result TantivyIndexerSettings { if let Some(b) = get_env_var("TANTIVY_MAX_INDEXED_JOB_LOG_SIZE__KB") { settings.max_indexed_job_log_size = (b * BYTES_PER_KB) as usize; } + if let Some(b) = get_env_var("TANTIVY_MAX_INDEX_TIME_WINDOW__S") { + settings.max_index_time_window_secs = b as i64; + } settings } diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 4de90be5c5..fb70db90ae 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -44,7 +44,7 @@ pub struct EnvRefWrapper { /// /// `Literal` serializes back to a plain JSON string, preserving backwards /// compatibility with existing consumers. -#[derive(Deserialize, Serialize, Clone, Debug)] +#[derive(Deserialize, Serialize, Clone)] #[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))] #[serde(untagged)] pub enum StringOrSecretRef { @@ -53,6 +53,16 @@ pub enum StringOrSecretRef { EnvRef(EnvRefWrapper), } +impl fmt::Debug for StringOrSecretRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Literal(_) => f.write_str("Literal(****)"), + Self::SecretRef(w) => f.debug_tuple("SecretRef").field(w).finish(), + Self::EnvRef(w) => f.debug_tuple("EnvRef").field(w).finish(), + } + } +} + impl StringOrSecretRef { /// Returns the literal string value, or `None` if this is an unresolved ref. pub fn as_literal(&self) -> Option<&str> { @@ -230,6 +240,8 @@ pub struct GlobalSettings { pub no_default_maven: Option, #[serde(skip_serializing_if = "Option::is_none")] pub default_tags_per_workspace: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub disable_hub: Option, // String settings #[serde(skip_serializing_if = "Option::is_none")] @@ -253,25 +265,25 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub instance_python_version: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub pip_index_url: Option, + pub pip_index_url: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub pip_extra_index_url: Option, + pub pip_extra_index_url: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub npm_config_registry: Option, + pub npm_config_registry: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub bunfig_install_scopes: Option, + pub bunfig_install_scopes: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub npmrc: Option, + pub npmrc: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub nuget_config: Option, + pub nuget_config: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub maven_repos: Option, + pub maven_repos: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub ruby_repos: Option, + pub ruby_repos: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub powershell_repo_url: Option, + pub powershell_repo_url: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub powershell_repo_pat: Option, + pub powershell_repo_pat: Option, // Array settings #[serde(skip_serializing_if = "Option::is_none")] @@ -922,7 +934,7 @@ fn redact_string(s: &str) -> String { } } -fn format_setting_value(key: &str, value: &serde_json::Value) -> String { +pub fn format_setting_value(key: &str, value: &serde_json::Value) -> String { if SENSITIVE_SETTINGS.contains(&key) { return match value { serde_json::Value::String(s) => format!("\"{}\"", redact_string(s)), @@ -2207,6 +2219,25 @@ mod tests { assert_eq!(v, *"world"); } + #[test] + fn string_or_secret_ref_debug_masks_literal() { + let v = StringOrSecretRef::Literal("super-secret-value".to_string()); + let debug = format!("{v:?}"); + assert_eq!(debug, "Literal(****)"); + assert!(!debug.contains("super-secret-value")); + } + + #[test] + fn format_setting_value_redacts_oauth_secrets() { + let val = serde_json::json!({ + "google": {"id": "client-id", "secret": "my-super-secret-12345"} + }); + let formatted = format_setting_value("oauths", &val); + assert!(!formatted.contains("my-super-secret-12345")); + assert!(formatted.contains("client-id")); + assert!(formatted.contains("****")); + } + #[test] #[should_panic(expected = "literal_value() called on unresolved secret ref")] fn string_or_secret_ref_literal_value_panics_on_ref() { diff --git a/backend/windmill-common/src/job_metrics.rs b/backend/windmill-common/src/job_metrics.rs index d416e4d44b..c49185de72 100644 --- a/backend/windmill-common/src/job_metrics.rs +++ b/backend/windmill-common/src/job_metrics.rs @@ -15,9 +15,11 @@ pub struct JobStatsRecord { pub timestamps: Option>>, pub timeseries_int: Option>, pub timeseries_float: Option>, + pub timeseries_start: Option>, + pub offsets_cs: Option>, } -#[derive(sqlx::Type, Debug, PartialEq, Deserialize, Serialize)] +#[derive(sqlx::Type, Debug, Clone, PartialEq, Deserialize, Serialize)] #[sqlx(type_name = "METRIC_KIND", rename_all = "snake_case")] pub enum MetricKind { ScalarInt, @@ -52,29 +54,21 @@ pub async fn register_metric_for_job( return Ok(metric_id); } - let (scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float) = match metric_kind - { + let is_timeseries = matches!( + metric_kind, + MetricKind::TimeseriesInt | MetricKind::TimeseriesFloat + ); + + let (scalar_int, scalar_float, timeseries_int, timeseries_float) = match metric_kind { MetricKind::ScalarInt | MetricKind::ScalarFloat => { - (None as Option, None as Option, None, None, None) + (None as Option, None as Option, None, None) } - MetricKind::TimeseriesInt => ( - None, - None, - Some(&[] as &[chrono::DateTime]), - Some(&[] as &[i32]), - None, - ), - MetricKind::TimeseriesFloat => ( - None, - None, - Some(&[] as &[chrono::DateTime]), - None, - Some(&[] as &[f32]), - ), + MetricKind::TimeseriesInt => (None, None, Some(&[] as &[i32]), None), + MetricKind::TimeseriesFloat => (None, None, None, Some(&[] as &[f32])), }; sqlx::query( - "INSERT INTO job_stats (workspace_id, job_id, metric_id, metric_name, metric_kind, scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + "INSERT INTO job_stats (workspace_id, job_id, metric_id, metric_name, metric_kind, scalar_int, scalar_float, timeseries_int, timeseries_float, timeseries_start, offsets_cs) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, CASE WHEN $10 THEN now() ELSE NULL END, CASE WHEN $10 THEN ARRAY[]::int[] ELSE NULL END)", ) .bind(workspace_id) .bind(job_id) @@ -83,9 +77,9 @@ pub async fn register_metric_for_job( .bind(metric_kind) .bind(scalar_int) .bind(scalar_float) - .bind(timestamps) .bind(timeseries_int) .bind(timeseries_float) + .bind(is_timeseries) .execute(db) .warn_after_seconds(1) .await?; @@ -117,6 +111,30 @@ pub async fn record_metric( } let metric_kind = metric_kind_opt.unwrap(); + record_metric_impl(db, workspace_id, job_id, metric_id, value, metric_kind).await +} + +/// Record a timeseries metric value without the extra SELECT to look up metric_kind. +/// Use this when the caller already knows the metric kind (e.g. the worker that registered it). +pub async fn record_timeseries_value( + db: &DB, + workspace_id: String, + job_id: Uuid, + metric_id: String, + value: MetricNumericValue, + metric_kind: MetricKind, +) -> error::Result<()> { + record_metric_impl(db, workspace_id, job_id, metric_id, value, metric_kind).await +} + +async fn record_metric_impl( + db: &DB, + workspace_id: String, + job_id: Uuid, + metric_id: String, + value: MetricNumericValue, + metric_kind: MetricKind, +) -> error::Result<()> { let (value_int, value_float) = match value { MetricNumericValue::Integer(val) => { if metric_kind != MetricKind::TimeseriesInt && metric_kind != MetricKind::ScalarInt { @@ -160,7 +178,7 @@ pub async fn record_metric( } MetricKind::TimeseriesInt => { sqlx::query!( - "UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", + "UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", &workspace_id, &job_id, &metric_id, @@ -169,7 +187,7 @@ pub async fn record_metric( } MetricKind::TimeseriesFloat => { sqlx::query!( - "UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", + "UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", &workspace_id, &job_id, &metric_id, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 643e08fc82..85f5563419 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -29,6 +29,7 @@ use sqlx::{Acquire, Postgres}; pub mod agent_workers; #[cfg(feature = "bedrock")] pub mod ai_bedrock; +pub mod ai_google; pub mod ai_providers; pub mod ai_types; pub mod apps; @@ -193,6 +194,7 @@ lazy_static::lazy_static! { pub static ref METRICS_DEBUG_ENABLED: AtomicBool = AtomicBool::new(false); pub static ref CRITICAL_ALERT_MUTE_UI_ENABLED: AtomicBool = AtomicBool::new(false); + pub static ref CRITICAL_ALERTS_ON_TOKEN_EXPIRY: AtomicBool = AtomicBool::new(false); pub static ref BASE_URL: Arc> = Arc::new(RwLock::new("".to_string())); pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index e4c5315cba..10b1a3b408 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -314,14 +314,14 @@ pub async fn create_directory_async(directory_path: &str) { .recursive(true) .create(directory_path) .await - .expect("could not create dir"); + .unwrap_or_else(|e| panic!("could not create dir '{}': {}", directory_path, e)); } pub fn create_directory_sync(directory_path: &str) { SyncDirBuilder::new() .recursive(true) .create(directory_path) - .expect("could not create dir"); + .unwrap_or_else(|e| panic!("could not create dir '{}': {}", directory_path, e)); } #[track_caller] diff --git a/backend/windmill-common/src/webhook.rs b/backend/windmill-common/src/webhook.rs index f1f8508f74..4677084fbe 100644 --- a/backend/windmill-common/src/webhook.rs +++ b/backend/windmill-common/src/webhook.rs @@ -39,29 +39,115 @@ pub enum WebhookPayload { #[serde(tag = "type")] pub enum WebhookMessage { // See https://serde.rs/enum-representations.html#internally-tagged for how this looks in JSON - CreateApp { workspace: String, path: String }, - DeleteApp { workspace: String, path: String }, - UpdateApp { workspace: String, old_path: String, new_path: String }, - CreateFlow { workspace: String, path: String }, - UpdateFlow { workspace: String, old_path: String, new_path: String }, - ArchiveFlow { workspace: String, path: String }, - DeleteFlow { workspace: String, path: String }, - CreateFolder { workspace: String, name: String }, - UpdateFolder { workspace: String, name: String }, - DeleteFolder { workspace: String, name: String }, - DeleteResource { workspace: String, path: String }, - CreateResource { workspace: String, path: String }, - UpdateResource { workspace: String, old_path: String, new_path: String }, - CreateResourceType { name: String }, - DeleteResourceType { name: String }, - UpdateResourceType { name: String }, - CreateScript { workspace: String, path: String, hash: String }, - UpdateScript { workspace: String, path: String, hash: String }, - DeleteScript { workspace: String, hash: String }, - DeleteScriptPath { workspace: String, path: String }, - CreateVariable { workspace: String, path: String }, - UpdateVariable { workspace: String, old_path: String, new_path: String }, - DeleteVariable { workspace: String, path: String }, + CreateApp { + workspace: String, + path: String, + }, + DeleteApp { + workspace: String, + path: String, + }, + UpdateApp { + workspace: String, + old_path: String, + new_path: String, + }, + CreateFlow { + workspace: String, + path: String, + }, + UpdateFlow { + workspace: String, + old_path: String, + new_path: String, + }, + ArchiveFlow { + workspace: String, + path: String, + }, + DeleteFlow { + workspace: String, + path: String, + }, + CreateFolder { + workspace: String, + name: String, + }, + UpdateFolder { + workspace: String, + name: String, + }, + DeleteFolder { + workspace: String, + name: String, + }, + DeleteResource { + workspace: String, + path: String, + }, + CreateResource { + workspace: String, + path: String, + }, + UpdateResource { + workspace: String, + old_path: String, + new_path: String, + }, + CreateResourceType { + name: String, + }, + DeleteResourceType { + name: String, + }, + UpdateResourceType { + name: String, + }, + CreateScript { + workspace: String, + path: String, + hash: String, + }, + UpdateScript { + workspace: String, + path: String, + hash: String, + }, + DeleteScript { + workspace: String, + hash: String, + }, + DeleteScriptPath { + workspace: String, + path: String, + }, + CreateVariable { + workspace: String, + path: String, + }, + UpdateVariable { + workspace: String, + old_path: String, + new_path: String, + }, + DeleteVariable { + workspace: String, + path: String, + }, + TokenExpiringSoon { + workspace: String, + token_prefix: String, + label: String, + owner: String, + expires_at: String, + days_remaining: i64, + }, + TokenExpired { + workspace: String, + token_prefix: String, + label: String, + owner: String, + }, } #[derive(Clone)] @@ -267,6 +353,20 @@ mod tests { new_path: "n".into(), }, WebhookMessage::DeleteVariable { workspace: "w".into(), path: "p".into() }, + WebhookMessage::TokenExpiringSoon { + workspace: "w".into(), + token_prefix: "abc1234567".into(), + label: "my-token".into(), + owner: "user@example.com".into(), + expires_at: "2026-03-10T00:00:00Z".into(), + days_remaining: 7, + }, + WebhookMessage::TokenExpired { + workspace: "w".into(), + token_prefix: "abc1234567".into(), + label: "my-token".into(), + owner: "user@example.com".into(), + }, ]; for msg in &messages { diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 8d95541f61..b3876fbcfa 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -354,6 +354,45 @@ impl HttpClient { ))) } } + + pub async fn get_bytes(&self, url: &str) -> anyhow::Result { + let base_url = self.base_internal_url.clone(); + let response = self + .client + .get(format!("{}{}", base_url, url)) + .send() + .await + .map_err(|e| anyhow::anyhow!(e))?; + if response.status().is_success() { + Ok(response.bytes().await?) + } else { + Err(anyhow::anyhow!( + "HTTP agent request GET {} failed {}", + url, + response.status() + )) + } + } + + pub async fn put_bytes(&self, url: &str, bytes: Bytes) -> anyhow::Result<()> { + let base_url = self.base_internal_url.clone(); + let response = self + .client + .put(format!("{}{}", base_url, url)) + .body(bytes) + .send() + .await + .map_err(|e| anyhow::anyhow!(e))?; + if response.status().is_success() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "HTTP agent request PUT {} failed {}", + url, + response.status() + )) + } + } } #[derive(Clone)] @@ -494,7 +533,17 @@ pub async fn store_pull_query(wc: &WorkerConfig) { lazy_static::lazy_static! { pub static ref WINDMILL_DIR: String = { let dir = std::env::var("WINDMILL_DIR") - .unwrap_or_else(|_| "/tmp/windmill".to_string()); + .unwrap_or_else(|_| { + #[cfg(not(windows))] + { "/tmp/windmill".to_string() } + #[cfg(windows)] + { + let temp = std::env::temp_dir(); + let temp_str = temp.to_string_lossy(); + let normalized = temp_str.trim_end_matches(&['/', '\\'][..]).replace('\\', "/"); + format!("{}/windmill", normalized) + } + }); if dir.is_empty() { panic!("WINDMILL_DIR must not be empty"); } @@ -688,6 +737,7 @@ pub struct PythonAnnotations { pub py311: bool, pub py312: bool, pub py313: bool, + pub sandbox: bool, } #[annotations("//")] @@ -701,6 +751,7 @@ pub struct TypeScriptAnnotations { pub nodejs: bool, pub native: bool, pub nobundling: bool, + pub sandbox: bool, } #[annotations("--")] @@ -2159,4 +2210,62 @@ mod tests { ); assert_ne!(a, b); } + + #[test] + fn test_python_sandbox_annotation() { + let content = "# sandbox\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); + } + + #[test] + fn test_python_sandbox_annotation_with_other_annotations() { + let content = "# no_cache\n# sandbox\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); + assert!(annotations.no_cache); + } + + #[test] + fn test_python_no_sandbox_annotation() { + let content = "# no_cache\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(!annotations.sandbox); + } + + #[test] + fn test_typescript_sandbox_annotation() { + let content = "// sandbox\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); + } + + #[test] + fn test_typescript_sandbox_annotation_with_other_annotations() { + let content = "// npm\n// sandbox\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); + assert!(annotations.npm); + } + + #[test] + fn test_typescript_no_sandbox_annotation() { + let content = "// npm\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(!annotations.sandbox); + } + + #[test] + fn test_python_sandbox_no_space() { + let content = "#sandbox\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); + } + + #[test] + fn test_typescript_sandbox_no_space() { + let content = "//sandbox\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); + } } diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index f56648f61e..5aba7162e1 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -146,6 +146,7 @@ pub enum ObjectType { Trigger, Settings, Key, + WorkspaceDependencies, } #[derive(Serialize, Deserialize, Debug)] diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index 645e477a62..f9cecd46ce 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -12,6 +12,10 @@ use windmill_common::{scripts::ScriptHash, DB}; pub mod git_sync_ee; pub mod git_sync_oss; +#[cfg(feature = "private")] +pub use git_sync_ee::{handle_deployment_metadata, handle_fork_branch_creation}; + +#[cfg(not(feature = "private"))] pub use git_sync_oss::{handle_deployment_metadata, handle_fork_branch_creation}; #[derive(Clone, Debug)] @@ -38,6 +42,7 @@ pub enum DeployedObject { EmailTrigger { path: String, parent_path: Option }, Settings { setting_type: String }, Key { key_type: String }, + WorkspaceDependencies { path: String }, } impl DeployedObject { @@ -65,6 +70,7 @@ impl DeployedObject { DeployedObject::EmailTrigger { path, .. } => path.to_owned(), DeployedObject::Settings { .. } => "settings.yaml".to_string(), DeployedObject::Key { .. } => "encryption_key.yaml".to_string(), + DeployedObject::WorkspaceDependencies { path, .. } => path.to_owned(), } } @@ -74,7 +80,8 @@ impl DeployedObject { | Self::Group { .. } | Self::ResourceType { .. } | Self::Settings { .. } - | Self::Key { .. } => true, + | Self::Key { .. } + | Self::WorkspaceDependencies { .. } => true, _ => false, } } @@ -103,6 +110,7 @@ impl DeployedObject { DeployedObject::EmailTrigger { parent_path, .. } => parent_path.to_owned(), DeployedObject::Settings { .. } => None, DeployedObject::Key { .. } => None, + DeployedObject::WorkspaceDependencies { .. } => None, } } @@ -130,6 +138,244 @@ impl DeployedObject { DeployedObject::EmailTrigger { .. } => "email_trigger", DeployedObject::Settings { .. } => "settings", DeployedObject::Key { .. } => "key", - }.to_string() + DeployedObject::WorkspaceDependencies { .. } => "workspace_dependencies", + } + .to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use windmill_common::scripts::ScriptHash; + + // --- DeployedObject::get_path tests --- + + #[test] + fn test_get_path_script() { + let obj = DeployedObject::Script { + hash: ScriptHash(123), + path: "f/folder/script".to_string(), + parent_path: None, + }; + assert_eq!(obj.get_path(), "f/folder/script"); + } + + #[test] + fn test_get_path_flow() { + let obj = DeployedObject::Flow { + path: "f/folder/flow".to_string(), + parent_path: Some("f/folder/old_flow".to_string()), + version: 1, + }; + assert_eq!(obj.get_path(), "f/folder/flow"); + } + + #[test] + fn test_get_path_user() { + let obj = DeployedObject::User { email: "user@example.com".to_string() }; + assert_eq!(obj.get_path(), "users/user@example.com"); + } + + #[test] + fn test_get_path_group() { + let obj = DeployedObject::Group { name: "admins".to_string() }; + assert_eq!(obj.get_path(), "groups/admins"); + } + + #[test] + fn test_get_path_settings() { + let obj = DeployedObject::Settings { setting_type: "error_handler".to_string() }; + assert_eq!(obj.get_path(), "settings.yaml"); + } + + #[test] + fn test_get_path_key() { + let obj = DeployedObject::Key { key_type: "encryption".to_string() }; + assert_eq!(obj.get_path(), "encryption_key.yaml"); + } + + #[test] + fn test_get_path_workspace_dependencies() { + let obj = DeployedObject::WorkspaceDependencies { + path: "workspace-dependencies/python".to_string(), + }; + assert_eq!(obj.get_path(), "workspace-dependencies/python"); + } + + // --- DeployedObject::get_ignore_regex_filter tests --- + + #[test] + fn test_ignore_regex_filter_user() { + let obj = DeployedObject::User { email: "user@example.com".to_string() }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_group() { + let obj = DeployedObject::Group { name: "admins".to_string() }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_resource_type() { + let obj = DeployedObject::ResourceType { path: "postgresql".to_string() }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_settings() { + let obj = DeployedObject::Settings { setting_type: "error_handler".to_string() }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_key() { + let obj = DeployedObject::Key { key_type: "encryption".to_string() }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_workspace_dependencies() { + let obj = DeployedObject::WorkspaceDependencies { + path: "workspace-dependencies/python".to_string(), + }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_script() { + let obj = DeployedObject::Script { + hash: ScriptHash(123), + path: "f/folder/script".to_string(), + parent_path: None, + }; + assert!(!obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_flow() { + let obj = DeployedObject::Flow { + path: "f/folder/flow".to_string(), + parent_path: None, + version: 1, + }; + assert!(!obj.get_ignore_regex_filter()); + } + + // --- DeployedObject::get_parent_path tests --- + + #[test] + fn test_get_parent_path_script_with_parent() { + let obj = DeployedObject::Script { + hash: ScriptHash(123), + path: "f/folder/script".to_string(), + parent_path: Some("f/folder/old_script".to_string()), + }; + assert_eq!(obj.get_parent_path(), Some("f/folder/old_script".to_string())); + } + + #[test] + fn test_get_parent_path_script_without_parent() { + let obj = DeployedObject::Script { + hash: ScriptHash(123), + path: "f/folder/script".to_string(), + parent_path: None, + }; + assert_eq!(obj.get_parent_path(), None); + } + + #[test] + fn test_get_parent_path_folder() { + let obj = DeployedObject::Folder { path: "f/folder".to_string() }; + assert_eq!(obj.get_parent_path(), None); + } + + #[test] + fn test_get_parent_path_workspace_dependencies() { + let obj = DeployedObject::WorkspaceDependencies { + path: "workspace-dependencies/python".to_string(), + }; + assert_eq!(obj.get_parent_path(), None); + } + + // --- DeployedObject::get_kind tests --- + + #[test] + fn test_get_kind_script() { + let obj = DeployedObject::Script { + hash: ScriptHash(123), + path: "test".to_string(), + parent_path: None, + }; + assert_eq!(obj.get_kind(), "script"); + } + + #[test] + fn test_get_kind_flow() { + let obj = DeployedObject::Flow { + path: "test".to_string(), + parent_path: None, + version: 1, + }; + assert_eq!(obj.get_kind(), "flow"); + } + + #[test] + fn test_get_kind_app() { + let obj = DeployedObject::App { + path: "test".to_string(), + version: 1, + parent_path: None, + }; + assert_eq!(obj.get_kind(), "app"); + } + + #[test] + fn test_get_kind_workspace_dependencies() { + let obj = DeployedObject::WorkspaceDependencies { + path: "workspace-dependencies/python".to_string(), + }; + assert_eq!(obj.get_kind(), "workspace_dependencies"); + } + + #[test] + fn test_get_kind_all_triggers() { + assert_eq!( + DeployedObject::HttpTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "http_trigger" + ); + assert_eq!( + DeployedObject::WebsocketTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "websocket_trigger" + ); + assert_eq!( + DeployedObject::KafkaTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "kafka_trigger" + ); + assert_eq!( + DeployedObject::NatsTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "nats_trigger" + ); + assert_eq!( + DeployedObject::PostgresTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "postgres_trigger" + ); + assert_eq!( + DeployedObject::MqttTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "mqtt_trigger" + ); + assert_eq!( + DeployedObject::SqsTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "sqs_trigger" + ); + assert_eq!( + DeployedObject::GcpTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "gcp_trigger" + ); + assert_eq!( + DeployedObject::EmailTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "email_trigger" + ); } } diff --git a/backend/windmill-jseval/src/lib.rs b/backend/windmill-jseval/src/lib.rs index 89feaf7b3e..7172e8cfc8 100644 --- a/backend/windmill-jseval/src/lib.rs +++ b/backend/windmill-jseval/src/lib.rs @@ -152,6 +152,21 @@ pub fn try_exact_property_access( None } +/// JS runtime properties (not methods) that cannot be resolved by PostgreSQL's +/// #> JSON path operator. Function calls like .map(...) already don't match the +/// RE_FULL regex due to parentheses, so only property accesses need listing here. +const JS_ONLY_PROPERTIES: &[&str] = &["length"]; + +fn ends_with_js_only_property(rest: Option<&str>) -> bool { + match rest { + None => false, + Some(rest) => { + let last_segment = rest.rsplit('.').next().unwrap_or(""); + JS_ONLY_PROPERTIES.contains(&last_segment) + } + } +} + pub async fn handle_full_regex( expr: &str, authed_client: &AuthedClient, @@ -162,6 +177,13 @@ pub async fn handle_full_regex( let obj_key = captures.get(2).unwrap().as_str(); let idx_o = captures.get(3).map(|y| y.as_str()); let rest = captures.get(4).map(|y| y.as_str()); + + // Skip the SQL fast path when the expression accesses a JS runtime + // property (e.g. .length) that the PostgreSQL #> operator can't resolve. + if ends_with_js_only_property(rest) { + return None; + } + let query = if let Some(idx) = idx_o { match rest { Some(rest) => Some(format!("{}{}", idx, rest)), diff --git a/backend/windmill-mcp/src/common/mod.rs b/backend/windmill-mcp/src/common/mod.rs index d67c1718cc..60ad18a85c 100644 --- a/backend/windmill-mcp/src/common/mod.rs +++ b/backend/windmill-mcp/src/common/mod.rs @@ -11,6 +11,8 @@ pub mod types; pub use schema::convert_schema_to_schema_type; pub use scope::{is_resource_allowed, parse_mcp_scopes, McpScopeConfig}; pub use transform::{ - apply_key_transformation, reverse_transform, reverse_transform_key, transform_path, + apply_key_transformation, extract_hub_version_id_from_hashed, + extract_path_prefix_from_hashed, parse_tool_prefix, reverse_transform, reverse_transform_key, + transform_hub_path, transform_path, }; pub use types::*; diff --git a/backend/windmill-mcp/src/common/transform.rs b/backend/windmill-mcp/src/common/transform.rs index 9a6fa317b1..0a12c85892 100644 --- a/backend/windmill-mcp/src/common/transform.rs +++ b/backend/windmill-mcp/src/common/transform.rs @@ -4,9 +4,15 @@ //! to make them compatible with MCP tool naming requirements. use super::types::SchemaType; +use windmill_common::utils::calculate_hash; -/// MCP clients do not allow names longer than 60 characters -const MAX_PATH_LENGTH: usize = 60; +/// Max tool name length. The MCP spec allows 64 chars, but some clients +/// (e.g. Cursor) prepend the server name to the tool name, so we use 40 +/// to leave room for that prefix. +const MAX_PATH_LENGTH: usize = 40; + +/// Length of the SHA256 hash suffix used for hashed names +const HASH_LEN: usize = 16; /// Transform the path for workspace scripts/flows /// @@ -14,19 +20,133 @@ const MAX_PATH_LENGTH: usize = 60; /// path with the type prefix. This is used when listing, because we can't /// have names with slashes. Because we replace slashes with underscores, /// we also need to escape underscores. +/// +/// For short names (≤40 chars): `s-{escaped_path}` or `f-{escaped_path}` +/// For long names (>40 chars): `S-{escaped[:22]}{sha256[:16]}` or `F-{escaped[:22]}{sha256[:16]}` +/// +/// The uppercase prefix signals that the name is hashed. pub fn transform_path(path: &str, type_str: &str) -> String { let escaped_path = path.replace('_', "__").replace('/', "_"); - // first letter of type_str is used as prefix, only one letter to avoid reaching 60 char name limit - let transformed_path = format!("{}-{}", &type_str[..1], escaped_path); - if transformed_path.len() > MAX_PATH_LENGTH { - let suffix = "_TRUNC"; - return format!( - "{}{}", - &transformed_path[..MAX_PATH_LENGTH - suffix.len()], - suffix - ); + let prefix_char = &type_str[..1]; + let short_name = format!("{}-{}", prefix_char, escaped_path); + + if short_name.len() <= MAX_PATH_LENGTH { + return short_name; } - transformed_path + + let upper_prefix = prefix_char.to_uppercase(); + // Layout: "{Upper}-" (2 chars) + prefix_body (22 chars) + hash (16 chars) = 40 + let prefix_body_len = MAX_PATH_LENGTH - 2 - HASH_LEN; + let hash = calculate_hash(&short_name); + let hash_suffix = &hash[..HASH_LEN]; + let truncated = truncate_to_char_boundary(&escaped_path, prefix_body_len); + format!("{}-{}{}", upper_prefix, truncated, hash_suffix) +} + +/// Transform the path for hub scripts +/// +/// For short names (≤40 chars): `hs-{id}-{summary}` +/// For long names (>40 chars): `Hs-{id}-{summary[:N]}{sha256[:16]}` +pub fn transform_hub_path(version_id: u64, summary: &str) -> String { + let escaped_summary = summary.replace(' ', "_"); + let short_name = format!("hs-{}-{}", version_id, escaped_summary); + + if short_name.len() <= MAX_PATH_LENGTH { + return short_name; + } + + let hash = calculate_hash(&short_name); + let hash_suffix = &hash[..HASH_LEN]; + // "Hs-{id}-" prefix, then fill remaining with summary + hash + let fixed_prefix = format!("Hs-{}-", version_id); + let available = MAX_PATH_LENGTH - fixed_prefix.len() - HASH_LEN; + let truncated_summary = truncate_to_char_boundary(&escaped_summary, available); + format!("{}{}{}", fixed_prefix, truncated_summary, hash_suffix) +} + +/// Parse the prefix of any tool name (both short and hashed). +/// Returns `(type_str, is_hub, is_hashed)`. +/// Hashed names use an uppercase first character as the signal. +pub fn parse_tool_prefix(name: &str) -> Result<(&str, bool, bool), String> { + let is_hashed = name.chars().next().map(|c| c.is_ascii_uppercase()).unwrap_or(false); + let lower = name.to_ascii_lowercase(); + let (type_str, is_hub) = if lower.starts_with("hs-") { + ("script", true) + } else if lower.starts_with("s-") { + ("script", false) + } else if lower.starts_with("f-") { + ("flow", false) + } else { + return Err(format!("Invalid tool name prefix: {}", name)); + }; + Ok((type_str, is_hub, is_hashed)) +} + +/// Extract the hub version_id from a hashed hub script name like `Hs-{id}-...` +pub fn extract_hub_version_id_from_hashed(name: &str) -> Result { + let rest = name + .strip_prefix("Hs-") + .ok_or_else(|| format!("Not a hashed hub name: {}", name))?; + let id = rest + .split('-') + .next() + .ok_or_else(|| format!("No version_id in hashed hub name: {}", name))?; + if id.is_empty() { + return Err(format!("Empty version_id in hashed hub name: {}", name)); + } + Ok(id.to_string()) +} + +/// Extract a safe original-path prefix from a hashed tool name. +/// +/// Given `S-u_admin_engineering__te`, extracts the escaped prefix between +/// the type prefix (`S-`, `F-`, or `Hs-`) and the hash, un-escapes it, and +/// returns a prefix suitable for `WHERE path LIKE '{prefix}%'`. +/// +/// Returns `None` if the name is too short or has an unrecognized prefix. +pub fn extract_path_prefix_from_hashed(name: &str) -> Option { + let prefix_len = if name.starts_with("Hs-") { + 3 + } else if name.starts_with("S-") || name.starts_with("F-") { + 2 + } else { + return None; + }; + if name.len() <= prefix_len + HASH_LEN { + return None; + } + let escaped_prefix = &name[prefix_len..name.len() - HASH_LEN]; + if escaped_prefix.is_empty() { + return None; + } + + // Strip trailing underscores — they may be half of a `__` pair split by truncation + let trimmed = escaped_prefix.trim_end_matches('_'); + if trimmed.is_empty() { + return None; + } + + Some(unescape_path(trimmed)) +} + +/// Un-escape a mangled path segment: `__` → `_`, standalone `_` → `/`. +fn unescape_path(s: &str) -> String { + const TEMP_PLACEHOLDER: &str = "@@UNDERSCORE@@"; + s.replace("__", TEMP_PLACEHOLDER) + .replace('_', "/") + .replace(TEMP_PLACEHOLDER, "_") +} + +/// Truncate a string to at most `max_len` bytes, ensuring we don't split a UTF-8 character. +fn truncate_to_char_boundary(s: &str, max_len: usize) -> &str { + if s.len() <= max_len { + return s; + } + let mut end = max_len; + while end > 0 && !s.is_char_boundary(end) { + end -= 1; + } + &s[..end] } /// Reverse the transformation of a path @@ -38,25 +158,22 @@ pub fn transform_path(path: &str, type_str: &str) -> String { /// This is used in call_tool to get the original path, and the type of the item. /// /// Returns: (type, original_path, is_hub) +/// +/// Note: This only works for non-hashed (short) names. Hashed names must be +/// resolved via `parse_tool_prefix` + path enumeration in the runner. pub fn reverse_transform(transformed_path: &str) -> Result<(&str, String, bool), String> { - let is_hub = transformed_path.starts_with("h"); - let transformed_path = if is_hub { - transformed_path[1..].to_string() - } else { - transformed_path.to_string() - }; - let type_str = if transformed_path.starts_with("s-") { - "script" - } else if transformed_path.starts_with("f-") { - "flow" - } else { - return Err(format!( - "Invalid prefix in transformed path: {}", - transformed_path - )); - }; + let (type_str, is_hub, is_hashed) = parse_tool_prefix(transformed_path)?; - let mangled_path = &transformed_path[2..]; + if is_hashed { + return Err( + "Hashed names cannot be reverse-transformed directly; use path enumeration instead" + .to_string(), + ); + } + + // Strip the prefix: "hs-" (3 chars) for hub, "s-"/"f-" (2 chars) for others + let prefix_len = if is_hub { 3 } else { 2 }; + let mangled_path = &transformed_path[prefix_len..]; let original_path = if is_hub { let parts = mangled_path.split("-").collect::>(); @@ -65,11 +182,7 @@ pub fn reverse_transform(transformed_path: &str) -> Result<(&str, String, bool), } parts[0].to_string() } else { - const TEMP_PLACEHOLDER: &str = "@@UNDERSCORE@@"; - mangled_path - .replace("__", TEMP_PLACEHOLDER) - .replace('_', "/") - .replace(TEMP_PLACEHOLDER, "_") + unescape_path(mangled_path) }; Ok((type_str, original_path, is_hub)) @@ -97,16 +210,13 @@ pub fn reverse_transform_key(transformed_key: &str, schema_obj: &Option s, None => { - // No schema available, return the key as is (best guess) return transformed_key.to_string(); } }; for original_key_in_schema in schema_obj.properties.keys() { - // Apply the SAME forward transformation to the schema key let potential_transformed_key = apply_key_transformation(original_key_in_schema); - // If it matches the key we received, we found the likely original if potential_transformed_key == transformed_key { return original_key_in_schema.clone(); } @@ -120,7 +230,7 @@ mod tests { use super::*; #[test] - fn test_transform_path() { + fn test_transform_path_short() { assert_eq!( transform_path("u/admin/script", "script"), "s-u_admin_script" @@ -130,7 +240,108 @@ mod tests { } #[test] - fn test_reverse_transform() { + fn test_transform_path_long_is_hashed() { + let long_path = "u/engineering/team/automation/very_long_script_name_that_exceeds_limit"; + let result = transform_path(long_path, "script"); + assert_eq!(result.len(), MAX_PATH_LENGTH); + assert!(result.starts_with("S-")); + let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap(); + assert!(is_hashed); + } + + #[test] + fn test_transform_path_long_flow_is_hashed() { + let long_path = "f/engineering/team/automation/very_long_flow_name_that_exceeds_limit"; + let result = transform_path(long_path, "flow"); + assert_eq!(result.len(), MAX_PATH_LENGTH); + assert!(result.starts_with("F-")); + let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap(); + assert!(is_hashed); + } + + #[test] + fn test_transform_path_hashing_is_deterministic() { + let path = "u/engineering/team/automation/very_long_script_name_that_exceeds_limit"; + let a = transform_path(path, "script"); + let b = transform_path(path, "script"); + assert_eq!(a, b); + } + + #[test] + fn test_transform_path_different_long_paths_differ() { + let a = transform_path( + "u/engineering/team/automation/very_long_script_name_that_exceeds_limit_a", + "script", + ); + let b = transform_path( + "u/engineering/team/automation/very_long_script_name_that_exceeds_limit_b", + "script", + ); + assert_ne!(a, b); + } + + #[test] + fn test_transform_hub_path_short() { + let result = transform_hub_path(12345, "Send Slack Message"); + assert_eq!(result, "hs-12345-Send_Slack_Message"); + let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap(); + assert!(!is_hashed); + } + + #[test] + fn test_transform_hub_path_long_is_hashed() { + let result = transform_hub_path( + 12345, + "Send Slack Message To Channel With Very Long Description That Exceeds Limit", + ); + assert_eq!(result.len(), MAX_PATH_LENGTH); + assert!(result.starts_with("Hs-12345-")); + let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap(); + assert!(is_hashed); + } + + #[test] + fn test_extract_hub_version_id_from_hashed() { + let name = "Hs-12345-Send_Slack_Message_To_Ch9e8d7c6b5a4f3e2d"; + let id = extract_hub_version_id_from_hashed(name).unwrap(); + assert_eq!(id, "12345"); + } + + #[test] + fn test_parse_tool_prefix() { + let (t, hub, hashed) = parse_tool_prefix("S-something").unwrap(); + assert_eq!(t, "script"); + assert!(!hub); + assert!(hashed); + + let (t, hub, hashed) = parse_tool_prefix("F-something").unwrap(); + assert_eq!(t, "flow"); + assert!(!hub); + assert!(hashed); + + let (t, hub, hashed) = parse_tool_prefix("Hs-12345-something").unwrap(); + assert_eq!(t, "script"); + assert!(hub); + assert!(hashed); + + let (t, hub, hashed) = parse_tool_prefix("s-u_admin_script").unwrap(); + assert_eq!(t, "script"); + assert!(!hub); + assert!(!hashed); + + let (t, hub, hashed) = parse_tool_prefix("f-f_folder_flow").unwrap(); + assert_eq!(t, "flow"); + assert!(!hub); + assert!(!hashed); + + let (t, hub, hashed) = parse_tool_prefix("hs-12345-summary").unwrap(); + assert_eq!(t, "script"); + assert!(hub); + assert!(!hashed); + } + + #[test] + fn test_reverse_transform_short_names() { let (type_str, path, is_hub) = reverse_transform("s-u_admin_script").unwrap(); assert_eq!(type_str, "script"); assert_eq!(path, "u/admin/script"); @@ -142,6 +353,70 @@ mod tests { assert!(!is_hub); } + #[test] + fn test_extract_path_prefix_from_hashed() { + // Generate a real hashed name and verify prefix extraction + let long_path = "u/admin/engineering/team/automation/very_long_script"; + let hashed = transform_path(long_path, "script"); + let (_, _, is_hashed) = parse_tool_prefix(&hashed).unwrap(); + assert!(is_hashed); + + let prefix = extract_path_prefix_from_hashed(&hashed).unwrap(); + // The original path should start with the extracted prefix + assert!( + long_path.starts_with(&prefix), + "path '{}' should start with prefix '{}'", + long_path, + prefix + ); + } + + #[test] + fn test_extract_path_prefix_underscore_in_path() { + let long_path = "u/admin/my_team/automation/very_long_script_name_here"; + let hashed = transform_path(long_path, "script"); + let prefix = extract_path_prefix_from_hashed(&hashed).unwrap(); + assert!( + long_path.starts_with(&prefix), + "path '{}' should start with prefix '{}'", + long_path, + prefix + ); + } + + #[test] + fn test_extract_path_prefix_rejects_invalid_prefix() { + assert!(extract_path_prefix_from_hashed("x-something").is_none()); + assert!(extract_path_prefix_from_hashed("").is_none()); + assert!(extract_path_prefix_from_hashed("S-").is_none()); + } + + #[test] + fn test_extract_path_prefix_handles_hs_prefix() { + // Hs- is 3 chars, not 2 — ensure the prefix is stripped correctly + let hashed = transform_hub_path(12345, "a]very long hub script summary that exceeds the limit"); + let (_, is_hub, is_hashed) = parse_tool_prefix(&hashed).unwrap(); + assert!(is_hub); + assert!(is_hashed); + + let prefix = extract_path_prefix_from_hashed(&hashed); + // Should not start with 's' (leftover from Hs- if sliced at index 2) + if let Some(ref p) = prefix { + assert!( + !p.starts_with('s'), + "prefix '{}' should not start with 's' from mis-sliced Hs- prefix", + p + ); + } + } + + #[test] + fn test_reverse_transform_rejects_hashed_names() { + assert!(reverse_transform("S-something").is_err()); + assert!(reverse_transform("F-something").is_err()); + assert!(reverse_transform("Hs-12345-something").is_err()); + } + #[test] fn test_apply_key_transformation() { assert_eq!(apply_key_transformation("my key"), "my_key"); diff --git a/backend/windmill-mcp/src/common/types.rs b/backend/windmill-mcp/src/common/types.rs index 43fe223ded..6161ca7963 100644 --- a/backend/windmill-mcp/src/common/types.rs +++ b/backend/windmill-mcp/src/common/types.rs @@ -92,8 +92,10 @@ pub struct ItemSchema { /// Trait for objects that can be converted to MCP tools pub trait ToolableItem { - /// Get the path or identifier for this item (transformed for MCP compatibility) - fn get_path_or_id(&self) -> String; + /// Get the MCP-compatible tool name (path transformed with escaping/hashing) + fn get_transformed_path(&self) -> String; + /// Get the original full path of this item (for display in tool title) + fn get_full_path(&self) -> &str; /// Get the summary/title of this item fn get_summary(&self) -> &str; /// Get the description of this item diff --git a/backend/windmill-mcp/src/lib.rs b/backend/windmill-mcp/src/lib.rs index a75b545154..7df6ee9f39 100644 --- a/backend/windmill-mcp/src/lib.rs +++ b/backend/windmill-mcp/src/lib.rs @@ -14,9 +14,9 @@ pub mod client; // Re-export common types at crate root for convenience pub use common::{ - convert_schema_to_schema_type, is_resource_allowed, parse_mcp_scopes, transform_path, FlowInfo, - HubResponse, HubScriptInfo, ItemSchema, McpScopeConfig, ResourceInfo, ResourceType, SchemaType, - ScriptInfo, ToolableItem, WorkspaceId, + convert_schema_to_schema_type, is_resource_allowed, parse_mcp_scopes, transform_hub_path, + transform_path, FlowInfo, HubResponse, HubScriptInfo, ItemSchema, McpScopeConfig, ResourceInfo, + ResourceType, SchemaType, ScriptInfo, ToolableItem, WorkspaceId, }; // Re-export client types at crate root for backward compatibility diff --git a/backend/windmill-mcp/src/server/backend.rs b/backend/windmill-mcp/src/server/backend.rs index fd01b2b26c..0b942353b3 100644 --- a/backend/windmill-mcp/src/server/backend.rs +++ b/backend/windmill-mcp/src/server/backend.rs @@ -55,20 +55,22 @@ pub trait McpBackend: Send + Sync + Clone + 'static { // Listing Operations // ───────────────────────────────────────────────────────────────── - /// List scripts, optionally filtered to favorites only + /// List scripts, optionally filtered to favorites only and/or by path prefix async fn list_scripts( &self, auth: &Self::Auth, workspace_id: &str, favorites_only: bool, + path_prefix: Option<&str>, ) -> BackendResult>; - /// List flows, optionally filtered to favorites only + /// List flows, optionally filtered to favorites only and/or by path prefix async fn list_flows( &self, auth: &Self::Auth, workspace_id: &str, favorites_only: bool, + path_prefix: Option<&str>, ) -> BackendResult>; /// List resource types in workspace diff --git a/backend/windmill-mcp/src/server/runner.rs b/backend/windmill-mcp/src/server/runner.rs index dd65c96334..be8764fa2e 100644 --- a/backend/windmill-mcp/src/server/runner.rs +++ b/backend/windmill-mcp/src/server/runner.rs @@ -5,7 +5,10 @@ use crate::common::schema::extract_resource_types_from_schema; use crate::common::scope::parse_mcp_scopes; -use crate::common::transform::{reverse_transform, reverse_transform_key}; +use crate::common::transform::{ + extract_hub_version_id_from_hashed, extract_path_prefix_from_hashed, parse_tool_prefix, + reverse_transform, reverse_transform_key, +}; use crate::common::types::{ResourceInfo, ToolableItem, WorkspaceId}; use crate::server::backend::{McpAuth, McpBackend}; use crate::server::endpoints::endpoint_tool_to_mcp_tool; @@ -81,6 +84,13 @@ impl Runner { } } +fn find_matching_path(candidates: Vec, request_name: &str) -> Option { + candidates + .into_iter() + .find(|item| item.get_transformed_path() == request_name) + .map(|item| item.get_full_path().to_string()) +} + impl ServerHandler for Runner { fn get_info(&self) -> ServerInfo { ServerInfo { @@ -120,9 +130,9 @@ impl ServerHandler for Runner { // Fetch all items concurrently let (scripts, flows, resource_types, hub_scripts) = tokio::try_join!( self.backend - .list_scripts(&auth, &workspace_id, favorites_only), + .list_scripts(&auth, &workspace_id, favorites_only, None), self.backend - .list_flows(&auth, &workspace_id, favorites_only), + .list_flows(&auth, &workspace_id, favorites_only, None), self.backend.list_resource_types(&auth, &workspace_id), async { if let Some(ref apps) = scope_config.hub_apps { @@ -231,17 +241,6 @@ impl ServerHandler for Runner { let scope_config = parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?; - // Handle truncated tool names - if request.name.ends_with("_TRUNC") { - return Ok(CallToolResult::error(vec![rmcp::model::Annotated::new( - rmcp::model::RawContent::Text(rmcp::model::RawTextContent { - text: "Tool path is too long. Consider shortening it to make it compatible with MCP.".to_string(), - meta: None, - }), - None, - )])); - } - let args = request.arguments.map(Value::Object).unwrap_or(Value::Null); // Check if this is an endpoint tool @@ -274,10 +273,58 @@ impl ServerHandler for Runner { } } - // Not an endpoint tool - parse as script/flow - let (tool_type, path, is_hub) = reverse_transform(&request.name).map_err(|e| { - ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None) - })?; + // Resolve the tool name to (type, path, is_hub) + let (type_str, is_hub, is_hashed) = + parse_tool_prefix(&request.name).map_err(|e| { + ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None) + })?; + + let (tool_type, path, is_hub) = if !is_hashed { + reverse_transform(&request.name).map_err(|e| { + ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None) + })? + } else if is_hub { + let version_id = + extract_hub_version_id_from_hashed(&request.name).map_err(|e| { + ErrorData::internal_error( + format!("Failed to extract hub version_id: {}", e), + None, + ) + })?; + (type_str, version_id, true) + } else { + let path_prefix = extract_path_prefix_from_hashed(&request.name); + let favorites_only = scope_config.favorites; + let matched_path = if type_str == "script" { + find_matching_path( + self.backend + .list_scripts(&auth, &workspace_id, favorites_only, path_prefix.as_deref()) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))?, + &request.name, + ) + } else { + find_matching_path( + self.backend + .list_flows(&auth, &workspace_id, favorites_only, path_prefix.as_deref()) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))?, + &request.name, + ) + }; + + let matched_path = matched_path.ok_or_else(|| { + ErrorData::internal_error( + format!( + "No {} found matching hashed tool name '{}'", + type_str, request.name + ), + None, + ) + })?; + + (type_str, matched_path, false) + }; // Validate script/flow scope if !is_hub && scope_config.granular { diff --git a/backend/windmill-mcp/src/server/tools.rs b/backend/windmill-mcp/src/server/tools.rs index a9262cdd6f..50ed03426b 100644 --- a/backend/windmill-mcp/src/server/tools.rs +++ b/backend/windmill-mcp/src/server/tools.rs @@ -9,7 +9,7 @@ use std::collections::HashMap; use std::sync::Arc; use crate::common::schema::{convert_schema_to_schema_type, make_schema_compatible}; -use crate::common::transform::transform_path; +use crate::common::transform::{transform_hub_path, transform_path}; use crate::common::types::{ FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, ToolableItem, }; @@ -17,10 +17,14 @@ use crate::server::backend::McpBackend; /// Implementation of ToolableItem for ScriptInfo impl ToolableItem for ScriptInfo { - fn get_path_or_id(&self) -> String { + fn get_transformed_path(&self) -> String { transform_path(&self.path, "script") } + fn get_full_path(&self) -> &str { + &self.path + } + fn get_summary(&self) -> &str { self.summary.as_deref().unwrap_or("No summary") } @@ -48,10 +52,14 @@ impl ToolableItem for ScriptInfo { /// Implementation of ToolableItem for FlowInfo impl ToolableItem for FlowInfo { - fn get_path_or_id(&self) -> String { + fn get_transformed_path(&self) -> String { transform_path(&self.path, "flow") } + fn get_full_path(&self) -> &str { + &self.path + } + fn get_summary(&self) -> &str { self.summary.as_deref().unwrap_or("No summary") } @@ -79,10 +87,13 @@ impl ToolableItem for FlowInfo { /// Implementation of ToolableItem for HubScriptInfo impl ToolableItem for HubScriptInfo { - fn get_path_or_id(&self) -> String { - let id = self.version_id; + fn get_transformed_path(&self) -> String { let summary = self.summary.as_deref().unwrap_or("No summary"); - format!("hs-{}-{}", id, summary.replace(" ", "_")) + transform_hub_path(self.version_id, summary) + } + + fn get_full_path(&self) -> &str { + self.summary.as_deref().unwrap_or("No summary") } fn get_summary(&self) -> &str { @@ -124,7 +135,7 @@ pub fn create_tool_from_item( resources_types: &[ResourceType], ) -> Tool { let is_hub = item.is_hub(); - let path = item.get_path_or_id(); + let path = item.get_transformed_path(); let item_type = item.item_type(); let description = format!( "This is a {} named `{}` with the following description: `{}`.{}", @@ -170,15 +181,24 @@ pub fn create_tool_from_item( } }; + let title = { + let summary = item.get_summary(); + if summary == "No summary" { + item.get_full_path().to_string() + } else { + summary.to_string() + } + }; + Tool { name: Cow::Owned(path), description: Some(Cow::Owned(description)), input_schema: Arc::new(input_schema_map), - title: Some(item.get_summary().to_string()), + title: Some(title.clone()), output_schema: None, icons: None, annotations: Some(ToolAnnotations { - title: Some(item.get_summary().to_string()), + title: Some(title), read_only_hint: Some(false), // Can modify environment destructive_hint: Some(true), // Can potentially be destructive idempotent_hint: Some(false), // Are not guaranteed to be idempotent diff --git a/backend/windmill-native-triggers/src/lib.rs b/backend/windmill-native-triggers/src/lib.rs index 4f27dff2e2..ed22480a9a 100644 --- a/backend/windmill-native-triggers/src/lib.rs +++ b/backend/windmill-native-triggers/src/lib.rs @@ -726,7 +726,7 @@ pub async fn get_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>> ) -> Result> { let token = sqlx::query_scalar!( r#" - SELECT token + SELECT token as "token!" FROM token WHERE token LIKE concat($1::text, '%') LIMIT 1 diff --git a/backend/windmill-oauth/src/lib.rs b/backend/windmill-oauth/src/lib.rs index e0c499c355..26d807dac3 100644 --- a/backend/windmill-oauth/src/lib.rs +++ b/backend/windmill-oauth/src/lib.rs @@ -94,7 +94,7 @@ pub struct OAuthConfig { } /// OAuth client credentials -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct OAuthClient { #[serde(default = "empty_string")] pub id: String, @@ -110,6 +110,21 @@ pub struct OAuthClient { pub grant_types: Vec, } +impl std::fmt::Debug for OAuthClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OAuthClient") + .field("id", &self.id) + .field("secret", &"***") + .field("display_name", &self.display_name) + .field("allowed_domains", &self.allowed_domains) + .field("connect_config", &self.connect_config) + .field("login_config", &self.login_config) + .field("tenant", &self.tenant) + .field("grant_types", &self.grant_types) + .finish() + } +} + fn empty_string() -> String { "".to_string() } @@ -608,7 +623,18 @@ pub async fn refresh_token<'c>( .await?; let account = windmill_common::utils::not_found_if_none(account, "Account", &id.to_string())?; - refresh_token_for_account(tx, path, w_id, id, db, account, oauth_clients, http_client, connect_configs_json).await + refresh_token_for_account( + tx, + path, + w_id, + id, + db, + account, + oauth_clients, + http_client, + connect_configs_json, + ) + .await } /// Refresh an OAuth token given pre-fetched account info (no additional SELECT). diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 806e3e99a2..d8cb3f4de4 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -818,7 +818,7 @@ pub async fn add_completed_job( flow_is_done: bool, duration: Option, from_cache: bool, -) -> Result<(Uuid, i64), Error> { +) -> Result<(Uuid, i64, Option), Error> { // tracing::error!("Start"); // let start = tokio::time::Instant::now(); @@ -830,7 +830,7 @@ pub async fn add_completed_job( } let result_columns = result_columns.as_ref(); - let (opt_uuid, duration, _skip_downstream_error_handlers) = (|| { + let (opt_uuid, duration, _skip_downstream_error_handlers, wac_job_ids) = (|| { commit_completed_job( db, completed_job, @@ -866,7 +866,7 @@ pub async fn add_completed_job( // if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout if let Some(job_id) = opt_uuid { - return Ok((job_id, duration)); + return Ok((job_id, duration, None)); } #[cfg(feature = "cloud")] @@ -887,7 +887,7 @@ pub async fn add_completed_job( // tracing::error!("4 {:?}", start.elapsed()); - Ok((completed_job.id, duration)) + Ok((completed_job.id, duration, wac_job_ids)) } async fn commit_completed_job( @@ -902,7 +902,7 @@ async fn commit_completed_job( flow_is_done: bool, duration: Option, from_cache: bool, -) -> windmill_common::error::Result<(Option, i64, bool)> { +) -> windmill_common::error::Result<(Option, i64, bool, Option)> { // let start = std::time::Instant::now(); let mut tx = db.begin().warn_after_seconds(10).await?; @@ -1003,25 +1003,31 @@ async fn commit_completed_job( .map_err(|e| Error::InternalErr(format!("Could not update job labels: {e:#}")))?; } + let mut wac_job_ids: Option = None; if !completed_job.is_flow_step() { if let Some(parent_job) = completed_job.parent_job { - let _ = sqlx::query_scalar!( - "UPDATE v2_job_status SET + // Only update WAC parents (v1 or v2). The WHERE condition skips + // non-WAC parents entirely (error handlers, run_script children, etc.). + // Also returns pending_steps.job_ids so WAC v2 child completion + // doesn't need a separate read. + let row = sqlx::query_scalar!( + r#"UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( jsonb_set( - COALESCE(workflow_as_code_status, '{}'::jsonb), + workflow_as_code_status, array[$1], COALESCE(workflow_as_code_status->$1, '{}'::jsonb) ), array[$1, 'duration_ms'], to_jsonb($2::bigint) ) - WHERE id = $3", + WHERE id = $3 AND workflow_as_code_status IS NOT NULL + RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS "job_ids: serde_json::Value""#, &completed_job.id.to_string(), duration, parent_job ) - .execute(&mut *tx) + .fetch_optional(&mut *tx) .warn_after_seconds(10) .await .inspect_err(|e| { @@ -1029,7 +1035,10 @@ async fn commit_completed_job( "Could not update parent job `duration_ms` in workflow as code status: {}", e, ) - }); + }) + .ok() + .flatten(); + wac_job_ids = row.flatten(); } } // tracing::error!("Added completed job {:#?}", queued_job); @@ -1250,14 +1259,14 @@ async fn commit_completed_job( completed_job.id ); // tracing::info!("completed job: {:?}", start.elapsed().as_micros()); - Ok((None, duration, _skip_downstream_error_handlers)) + Ok((None, duration, _skip_downstream_error_handlers, wac_job_ids)) } async fn check_result_size( db: &Pool, queued_job: &MiniCompletedJob, result: Json<&T>, -) -> Option, i64, bool), Error>> { +) -> Option, i64, bool, Option), Error>> { let result_size = result.size() / 1024 / 1024; if result_size > 2 { if result_size > *MAX_RESULT_SIZE_MB { diff --git a/backend/windmill-test-utils/Cargo.toml b/backend/windmill-test-utils/Cargo.toml index e805ee5a3d..d1729a7511 100644 --- a/backend/windmill-test-utils/Cargo.toml +++ b/backend/windmill-test-utils/Cargo.toml @@ -10,8 +10,8 @@ path = "src/lib.rs" [features] default = [] -private = [] -enterprise = [] +private = ["windmill-api/private"] +enterprise = ["windmill-api/enterprise"] python = ["windmill-common/python"] deno_core = ["dep:windmill-runtime-nativets"] agent_worker_server = ["dep:windmill-api-agent-workers"] diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index 69e7fc0558..5d5c80b470 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -66,6 +66,8 @@ fn next_worker_name() -> String { .unwrap_or(s) }) .unwrap_or("no thread name"); + // Replace colons because they are illegal in Windows directory names + let thread_name = thread_name.replace(':', "_"); format!("{id}/worker-{thread_name}") } @@ -149,6 +151,7 @@ pub struct RunJob { pub args: serde_json::Map, pub scheduled_for_o: Option>, pub email: String, + pub job_id: Option, } impl From for RunJob { @@ -158,6 +161,7 @@ impl From for RunJob { args: Default::default(), scheduled_for_o: None, email: "test@windmill.dev".to_string(), + job_id: None, } } } @@ -181,8 +185,13 @@ impl RunJob { self } + pub fn job_id(mut self, id: Uuid) -> Self { + self.job_id = Some(id); + self + } + pub async fn push(self, db: &Pool) -> Uuid { - let RunJob { payload, args, scheduled_for_o, email } = self; + let RunJob { payload, args, scheduled_for_o, email, job_id } = self; let mut hm_args = std::collections::HashMap::new(); for (k, v) in args { hm_args.insert(k, windmill_common::worker::to_raw_value(&v)); @@ -204,7 +213,7 @@ impl RunJob { /* parent_job */ None, /* root job */ None, /* flow_innermost_root_job */ None, - /* job_id */ None, + /* job_id */ job_id, /* is_flow_step */ false, /* same_worker */ false, None, @@ -228,7 +237,7 @@ impl RunJob { /// Push the job as a specific user (for testing permissions) pub async fn push_as(self, db: &Pool, username: &str, email: &str) -> Uuid { - let RunJob { payload, args, scheduled_for_o, .. } = self; + let RunJob { payload, args, scheduled_for_o, job_id, .. } = self; let mut hm_args = std::collections::HashMap::new(); for (k, v) in args { hm_args.insert(k, windmill_common::worker::to_raw_value(&v)); @@ -250,7 +259,7 @@ impl RunJob { /* parent_job */ None, /* root job */ None, /* flow_innermost_root_job */ None, - /* job_id */ None, + /* job_id */ job_id, /* is_flow_step */ false, /* same_worker */ false, None, @@ -828,6 +837,15 @@ pub async fn run_preview_relative_imports( #[cfg(all(feature = "private", feature = "agent_worker_server"))] pub async fn testing_http_connection(port: u16) -> Connection { + testing_http_connection_with_tags( + port, + vec!["flow".into(), "python3".into(), "dependency".into()], + ) + .await +} + +#[cfg(all(feature = "private", feature = "agent_worker_server"))] +pub async fn testing_http_connection_with_tags(port: u16, tags: Vec) -> Connection { let suffix = windmill_common::utils::create_default_worker_suffix("test-agent-worker"); let agent_token = format!( "{}{}", @@ -835,7 +853,7 @@ pub async fn testing_http_connection(port: u16) -> Connection { windmill_common::jwt::encode_with_internal_secret(windmill_api_agent_workers::AgentAuth { worker_group: "testing-agent".to_owned(), suffix: Some(suffix.clone()), - tags: vec!["flow".into(), "python3".into(), "dependency".into()], + tags, exp: Some(usize::MAX), }) .await diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs index cf86d83801..be20cfae3e 100644 --- a/backend/windmill-types/src/assets.rs +++ b/backend/windmill-types/src/assets.rs @@ -13,6 +13,7 @@ pub enum AssetKind { Variable, // Deprecated Ducklake, DataTable, + Volume, } #[derive( diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index bcbe0ede33..28d3a1e449 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -105,15 +105,15 @@ impl ScriptLang { pub fn is_native(&self) -> bool { matches!( self, - ScriptLang::Bunnative | - ScriptLang::Nativets | - ScriptLang::Postgresql | - ScriptLang::Mysql | - ScriptLang::Graphql | - ScriptLang::Snowflake | - ScriptLang::Mssql | - ScriptLang::Bigquery | - ScriptLang::OracleDB + ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Postgresql + | ScriptLang::Mysql + | ScriptLang::Graphql + | ScriptLang::Snowflake + | ScriptLang::Mssql + | ScriptLang::Bigquery + | ScriptLang::OracleDB ) } @@ -459,6 +459,7 @@ pub struct NewScript { pub path: String, pub parent_hash: Option, pub summary: String, + #[serde(default)] pub description: String, pub content: String, pub schema: Option, diff --git a/backend/windmill-worker-volumes/Cargo.toml b/backend/windmill-worker-volumes/Cargo.toml new file mode 100644 index 0000000000..08b9a554b2 --- /dev/null +++ b/backend/windmill-worker-volumes/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "windmill-worker-volumes" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_worker_volumes" +path = "src/lib.rs" + +[features] +enterprise = [] +private = [] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +object_store.workspace = true +tokio.workspace = true +tracing.workspace = true +bytes.workspace = true +futures.workspace = true +serde.workspace = true +serde_json.workspace = true +regex.workspace = true +lazy_static.workspace = true +md-5.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/backend/windmill-worker-volumes/src/lib.rs b/backend/windmill-worker-volumes/src/lib.rs new file mode 100644 index 0000000000..a9003a3712 --- /dev/null +++ b/backend/windmill-worker-volumes/src/lib.rs @@ -0,0 +1,544 @@ +#[cfg(feature = "private")] +mod volume_ee; +mod volume_oss; +pub use volume_oss::*; + +pub use object_store::ObjectStore as DynObjectStore; + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::collections::HashSet; +use std::path::PathBuf; + +pub const MAX_VOLUMES_PER_JOB: usize = 10; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FileEntry { + pub size: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub md5: Option, +} + +pub fn compute_md5_hex(data: &[u8]) -> String { + use md5::{Digest, Md5}; + let result = Md5::digest(data); + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut hex = String::with_capacity(32); + for &b in result.iter() { + hex.push(HEX[(b >> 4) as usize] as char); + hex.push(HEX[(b & 0x0f) as usize] as char); + } + hex +} + +/// Extract an MD5 hash from an S3 ETag, if it's a simple (non-multipart) ETag. +pub fn etag_to_md5(e_tag: Option<&str>) -> Option { + let tag = e_tag?.trim_matches('"'); + // Multipart ETags contain a '-' (e.g. "abc123-5"), skip those + if tag.contains('-') || tag.is_empty() { + return None; + } + Some(tag.to_string()) +} + +lazy_static::lazy_static! { + static ref ARGS_INTERPOLATION_RE: regex::Regex = + regex::Regex::new(r#"\$args\[((?:\w+\.)*\w+)\]"#).unwrap(); + static ref VALID_VOLUME_NAME_RE: regex::Regex = + regex::Regex::new(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,253}[a-zA-Z0-9]$").unwrap(); +} + +#[derive(Debug, Clone, PartialEq)] +pub struct VolumeMount { + pub name: String, + pub target: String, +} + +pub struct VolumeState { + pub mount: VolumeMount, + pub local_dir: PathBuf, + pub manifest: HashMap, + pub symlinks: HashMap, +} + +pub struct DownloadStats { + pub total_files: usize, + pub from_cache: usize, + pub downloaded: usize, +} + +pub struct SyncStats { + pub new_size_bytes: u64, + pub file_count: usize, + pub uploaded: usize, + pub skipped: usize, +} + +pub fn validate_volume_name(name: &str) -> Result<(), String> { + if name.contains("..") { + return Err(format!( + "Volume name '{}' contains '..' which is not allowed", + name + )); + } + if !VALID_VOLUME_NAME_RE.is_match(name) { + return Err(format!( + "Volume name '{}' is invalid. Names must be 2-255 characters, \ + start and end with alphanumeric, and contain only alphanumeric, '.', '_', or '-'", + name + )); + } + Ok(()) +} + +const ALLOWED_ABSOLUTE_PREFIXES: &[&str] = &["/tmp/", "/mnt/", "/opt/", "/home/", "/data/"]; + +pub fn validate_volume_target(target: &str) -> Result<(), String> { + if target.split('/').any(|seg| seg == "..") { + return Err(format!( + "Volume target '{target}' contains '..' segments which is not allowed" + )); + } + if target.starts_with('/') + && !ALLOWED_ABSOLUTE_PREFIXES + .iter() + .any(|p| target.starts_with(p)) + { + return Err(format!( + "Volume target '{target}' must be a relative path or start with one of: {}", + ALLOWED_ABSOLUTE_PREFIXES.join(", ") + )); + } + Ok(()) +} + +pub fn validate_volume_mounts(mounts: &[VolumeMount]) -> Result<(), String> { + if mounts.len() > MAX_VOLUMES_PER_JOB { + return Err(format!( + "Too many volume mounts ({}, max {})", + mounts.len(), + MAX_VOLUMES_PER_JOB + )); + } + let mut seen_names = HashSet::new(); + let mut seen_targets = HashSet::new(); + for v in mounts { + if !seen_names.insert(&v.name) { + return Err(format!("Duplicate volume name: '{}'", v.name)); + } + if !seen_targets.insert(&v.target) { + return Err(format!("Duplicate volume target: '{}'", v.target)); + } + } + Ok(()) +} + +pub fn interpolate_volume_name( + name: &str, + args: Option<&HashMap>>, + workspace_id: &str, +) -> String { + let name = name.replace("$workspace", workspace_id); + if !name.contains("$args[") { + return name; + } + let Some(args) = args else { + return name; + }; + let mut result = name.clone(); + for cap in ARGS_INTERPOLATION_RE.captures_iter(&name) { + let full_match = cap.get(0).unwrap().as_str(); + let arg_name = cap.get(1).unwrap().as_str(); + let arg_value = if arg_name.contains('.') { + let parts: Vec<&str> = arg_name.split('.').collect(); + let root = parts[0]; + let mut value = args + .get(root) + .map(|x| x.get().to_string()) + .unwrap_or_default(); + for part in parts.iter().skip(1) { + if let Ok(obj) = serde_json::from_str::(&value) { + value = obj + .get(part) + .map(|v| v.to_string()) + .unwrap_or_default() + .to_string(); + } else { + value = String::new(); + break; + } + } + value.trim_matches('"').to_string() + } else { + args.get(arg_name) + .map(|x| x.get().trim_matches('"').to_string()) + .unwrap_or_default() + }; + result = result.replace(full_match, &arg_value); + } + result +} + +pub fn parse_volume_annotations(content: &str, comment_prefix: &str) -> Vec { + let mut volumes = Vec::new(); + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + if !trimmed.starts_with(comment_prefix) { + break; + } + let after_prefix = trimmed[comment_prefix.len()..].trim(); + if let Some(rest) = after_prefix.strip_prefix("volume:") { + let rest = rest.trim(); + let mut parts = rest.splitn(2, char::is_whitespace); + if let (Some(name), Some(target)) = (parts.next(), parts.next()) { + let name = name.trim(); + let target = target.trim(); + if !name.is_empty() && !target.is_empty() { + volumes + .push(VolumeMount { name: name.to_string(), target: target.to_string() }); + } + } + } + } + volumes +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_python_single_volume() { + let content = "# volume: mydata /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_typescript_single_volume() { + let content = "// volume: mydata /tmp/data\nexport function main() {}"; + let result = parse_volume_annotations(content, "//"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_multiple_volumes() { + let content = "# volume: data1 /tmp/data1\n# volume: data2 /tmp/data2\n# volume: models /opt/models\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![ + VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }, + VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() }, + VolumeMount { name: "models".to_string(), target: "/opt/models".to_string() }, + ] + ); + } + + #[test] + fn parse_mixed_annotations_and_volumes() { + let content = "# sandbox\n# volume: mydata /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_no_volumes() { + let content = "# sandbox\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert!(result.is_empty()); + } + + #[test] + fn parse_empty_content() { + let result = parse_volume_annotations("", "#"); + assert!(result.is_empty()); + } + + #[test] + fn parse_stops_at_non_comment_line() { + let content = + "# volume: data1 /tmp/data1\ndef main():\n # volume: data2 /tmp/data2\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }] + ); + } + + #[test] + fn parse_skips_blank_lines_in_header() { + let content = + "# volume: data1 /tmp/data1\n\n# volume: data2 /tmp/data2\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![ + VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }, + VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() }, + ] + ); + } + + #[test] + fn parse_ignores_malformed_volume_lines() { + let content = + "# volume:\n# volume: onlyname\n# volume: good /tmp/good\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "good".to_string(), target: "/tmp/good".to_string() }] + ); + } + + #[test] + fn parse_extra_whitespace() { + let content = "# volume: mydata /tmp/data \ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_target_with_spaces_in_path() { + let content = "// volume: mydata /tmp/my data dir\nexport function main() {}"; + let result = parse_volume_annotations(content, "//"); + assert_eq!( + result, + vec![VolumeMount { + name: "mydata".to_string(), + target: "/tmp/my data dir".to_string(), + }] + ); + } + + #[test] + fn parse_volume_with_dashes_and_underscores() { + let content = "# volume: my-data_v2 /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "my-data_v2".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn interpolate_workspace() { + let name = "$workspace-data"; + let result = interpolate_volume_name(name, None, "my_ws"); + assert_eq!(result, "my_ws-data"); + } + + #[test] + fn interpolate_args_simple() { + let mut args = HashMap::new(); + args.insert( + "env".to_string(), + serde_json::value::RawValue::from_string("\"prod\"".to_string()).unwrap(), + ); + let result = interpolate_volume_name("data-$args[env]", Some(&args), "ws"); + assert_eq!(result, "data-prod"); + } + + #[test] + fn interpolate_args_and_workspace() { + let mut args = HashMap::new(); + args.insert( + "env".to_string(), + serde_json::value::RawValue::from_string("\"staging\"".to_string()).unwrap(), + ); + let result = interpolate_volume_name("$workspace-$args[env]-cache", Some(&args), "acme"); + assert_eq!(result, "acme-staging-cache"); + } + + #[test] + fn interpolate_no_placeholders() { + let result = interpolate_volume_name("plain-name", None, "ws"); + assert_eq!(result, "plain-name"); + } + + #[test] + fn interpolate_missing_arg() { + let args = HashMap::new(); + let result = interpolate_volume_name("data-$args[missing]", Some(&args), "ws"); + assert_eq!(result, "data-"); + } + + #[test] + fn interpolate_nested_arg() { + let mut args = HashMap::new(); + args.insert( + "config".to_string(), + serde_json::value::RawValue::from_string( + r#"{"env": "prod", "region": "us-east"}"#.to_string(), + ) + .unwrap(), + ); + let result = interpolate_volume_name( + "data-$args[config.env]-$args[config.region]", + Some(&args), + "ws", + ); + assert_eq!(result, "data-prod-us-east"); + } + + #[test] + fn parse_wrong_prefix_returns_empty() { + let content = "# volume: mydata /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "//"); + assert!(result.is_empty()); + } + + #[test] + fn parse_relative_path() { + let content = "// volume: agent-memory .claude\nexport function main() {}"; + let result = parse_volume_annotations(content, "//"); + assert_eq!( + result, + vec![VolumeMount { name: "agent-memory".to_string(), target: ".claude".to_string() }] + ); + } + + #[test] + fn parse_relative_nested_path() { + let content = "# volume: data data/models\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "data".to_string(), target: "data/models".to_string() }] + ); + } + + #[test] + fn validate_valid_names() { + assert!(validate_volume_name("mydata").is_ok()); + assert!(validate_volume_name("my-data_v2").is_ok()); + assert!(validate_volume_name("acme-staging-cache").is_ok()); + assert!(validate_volume_name("a1").is_ok()); + assert!(validate_volume_name("data.v2").is_ok()); + assert!(validate_volume_name("A0").is_ok()); + } + + #[test] + fn validate_rejects_path_traversal() { + assert!(validate_volume_name("../other-workspace").is_err()); + assert!(validate_volume_name("data/../secrets").is_err()); + assert!(validate_volume_name("a..b").is_err()); + } + + #[test] + fn validate_rejects_special_start_end() { + assert!(validate_volume_name("-data").is_err()); + assert!(validate_volume_name("data-").is_err()); + assert!(validate_volume_name(".data").is_err()); + assert!(validate_volume_name("data.").is_err()); + assert!(validate_volume_name("_data").is_err()); + } + + #[test] + fn validate_rejects_path_separators() { + assert!(validate_volume_name("data/secrets").is_err()); + assert!(validate_volume_name("data\\secrets").is_err()); + } + + #[test] + fn validate_rejects_too_short() { + assert!(validate_volume_name("").is_err()); + assert!(validate_volume_name("a").is_err()); + } + + #[test] + fn validate_rejects_too_long() { + let long_name = format!("a{}a", "b".repeat(254)); + assert!(validate_volume_name(&long_name).is_err()); + } + + #[test] + fn validate_rejects_spaces_and_special() { + assert!(validate_volume_name("my data").is_err()); + assert!(validate_volume_name("my@data").is_err()); + assert!(validate_volume_name("my$data").is_err()); + } + + #[test] + fn validate_target_allows_relative() { + assert!(validate_volume_target("data").is_ok()); + assert!(validate_volume_target("data/models").is_ok()); + assert!(validate_volume_target(".claude").is_ok()); + } + + #[test] + fn validate_target_allows_safe_absolute() { + assert!(validate_volume_target("/tmp/data").is_ok()); + assert!(validate_volume_target("/mnt/data").is_ok()); + assert!(validate_volume_target("/opt/models").is_ok()); + assert!(validate_volume_target("/home/user/data").is_ok()); + assert!(validate_volume_target("/data/cache").is_ok()); + } + + #[test] + fn validate_target_rejects_dangerous_absolute() { + assert!(validate_volume_target("/etc/passwd").is_err()); + assert!(validate_volume_target("/proc/self").is_err()); + assert!(validate_volume_target("/sys/fs").is_err()); + assert!(validate_volume_target("/dev/null").is_err()); + assert!(validate_volume_target("/usr/bin").is_err()); + assert!(validate_volume_target("/var/log").is_err()); + } + + #[test] + fn validate_target_rejects_traversal() { + assert!(validate_volume_target("../../etc").is_err()); + assert!(validate_volume_target("data/../../../etc").is_err()); + assert!(validate_volume_target("/tmp/../etc/passwd").is_err()); + } + + #[test] + fn validate_mounts_rejects_too_many() { + let mounts: Vec = (0..11) + .map(|i| VolumeMount { name: format!("v{:02}", i), target: format!("t{}", i) }) + .collect(); + assert!(validate_volume_mounts(&mounts).is_err()); + } + + #[test] + fn validate_mounts_rejects_duplicate_name() { + let mounts = vec![ + VolumeMount { name: "data".to_string(), target: "/tmp/a".to_string() }, + VolumeMount { name: "data".to_string(), target: "/tmp/b".to_string() }, + ]; + assert!(validate_volume_mounts(&mounts).is_err()); + } + + #[test] + fn validate_mounts_rejects_duplicate_target() { + let mounts = vec![ + VolumeMount { name: "v1".to_string(), target: "/tmp/data".to_string() }, + VolumeMount { name: "v2".to_string(), target: "/tmp/data".to_string() }, + ]; + assert!(validate_volume_mounts(&mounts).is_err()); + } + + #[test] + fn validate_mounts_ok() { + let mounts = vec![ + VolumeMount { name: "v1".to_string(), target: "/tmp/a".to_string() }, + VolumeMount { name: "v2".to_string(), target: "/tmp/b".to_string() }, + ]; + assert!(validate_volume_mounts(&mounts).is_ok()); + } +} diff --git a/backend/windmill-worker-volumes/src/volume_oss.rs b/backend/windmill-worker-volumes/src/volume_oss.rs new file mode 100644 index 0000000000..07cd6906d2 --- /dev/null +++ b/backend/windmill-worker-volumes/src/volume_oss.rs @@ -0,0 +1,116 @@ +#[cfg(feature = "private")] +pub use crate::volume_ee::*; + +#[cfg(not(feature = "private"))] +use crate::{DownloadStats, SyncStats, VolumeMount, VolumeState}; +#[cfg(not(feature = "private"))] +use object_store::ObjectStore; +#[cfg(not(feature = "private"))] +use std::path::Path; +#[cfg(not(feature = "private"))] +use std::sync::Arc; +#[cfg(not(feature = "private"))] +use windmill_common::error; + +#[cfg(not(feature = "private"))] +pub async fn download_volume( + _client: Arc, + _volume: &VolumeMount, + _job_dir: &str, + _workspace_id: &str, +) -> error::Result<(VolumeState, DownloadStats)> { + Err(error::Error::internal_err( + "Volumes are not available in this build".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +pub fn volume_nsjail_mount(_local_dir: &Path, _target: &str) -> String { + String::new() +} + +#[cfg(not(feature = "private"))] +pub async fn sync_volume_back( + _client: Arc, + _state: &VolumeState, + _workspace_id: &str, +) -> error::Result { + Err(error::Error::internal_err( + "Volumes are not available in this build".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +pub fn walk_dir(dir: &Path) -> std::io::Result> { + let mut result = Vec::new(); + walk_dir_inner(dir, &mut result)?; + Ok(result) +} + +#[cfg(not(feature = "private"))] +fn walk_dir_inner(dir: &Path, result: &mut Vec) -> std::io::Result<()> { + if !dir.is_dir() { + return Ok(()); + } + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + let meta = match std::fs::symlink_metadata(&path) { + Ok(m) => m, + Err(_) => continue, + }; + if meta.is_dir() { + walk_dir_inner(&path, result)?; + } else if meta.is_file() { + result.push(path); + } + } + Ok(()) +} + +#[cfg(not(feature = "private"))] +pub fn collect_symlinks(dir: &Path) -> std::collections::HashMap { + let mut symlinks = std::collections::HashMap::new(); + collect_symlinks_inner(dir, dir, &mut symlinks); + symlinks +} + +#[cfg(not(feature = "private"))] +fn collect_symlinks_inner( + base: &Path, + dir: &Path, + symlinks: &mut std::collections::HashMap, +) { + let entries = match std::fs::read_dir(dir) { + Ok(e) => e, + Err(_) => return, + }; + for entry in entries { + let entry = match entry { + Ok(e) => e, + Err(_) => continue, + }; + let path = entry.path(); + let meta = match std::fs::symlink_metadata(&path) { + Ok(m) => m, + Err(_) => continue, + }; + if meta.file_type().is_symlink() { + if let Ok(target) = std::fs::read_link(&path) { + let relative = path + .strip_prefix(base) + .unwrap_or(&path) + .to_string_lossy() + .replace('\\', "/"); + symlinks.insert(relative, target.to_string_lossy().to_string()); + } + } else if meta.is_dir() { + collect_symlinks_inner(base, &path, symlinks); + } + } +} + +#[cfg(not(feature = "private"))] +pub fn restore_symlinks(_dir: &Path, _symlinks: &std::collections::HashMap) { + // No-op in OSS build +} diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 5cf0a4c26e..23927753ee 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -10,10 +10,10 @@ path = "src/lib.rs" [features] default = [] -private = [] +private = ["windmill-worker-volumes/private", "windmill-queue/private"] mcp = ["dep:windmill-mcp"] prometheus = ["dep:prometheus", "windmill-common/prometheus"] -enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] +enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker-volumes/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] mssql = ["dep:tiberius"] mssql-kerberos = ["mssql", "tiberius/integrated-auth-gssapi"] # Linux/Unix integrated auth mssql-winauth = ["mssql", "tiberius/winauth"] # Windows integrated auth @@ -47,6 +47,7 @@ windmill-audit.workspace = true # there isn't really a reason for audit-worth ac windmill-common = { workspace = true, default-features = false } windmill-types.workspace = true windmill-object-store.workspace = true +windmill-worker-volumes.workspace = true windmill-jseval.workspace = true windmill-runtime-nativets = { workspace = true, optional = true } windmill-mcp = { workspace = true, optional = true } diff --git a/backend/windmill-worker/loader.bun.windows.js b/backend/windmill-worker/loader.bun.windows.js new file mode 100644 index 0000000000..227c68a56c --- /dev/null +++ b/backend/windmill-worker/loader.bun.windows.js @@ -0,0 +1,123 @@ +// Windows-specific bun loader that uses a virtual "windmill-url" namespace instead +// of writing .url files to disk. This avoids Windows path issues (backslashes in +// resolve(), 8.3 short filenames, drive letter prefixes). The virtual namespace +// approach is likely better on all fronts but we keep the original .url-file loader +// on Linux to avoid breaking back-compat. +const p = { + name: "windmill-relative-resolver", + async setup(build) { + const { readFileSync } = await import("fs"); + const { resolve } = await import("node:path"); + + const base_internal_url = "BASE_INTERNAL_URL".replace( + "localhost", + "127.0.0.1" + ); + + const w_id = "W_ID"; + const current_path = "CURRENT_PATH"; + const token = "TOKEN"; + + const cdir = resolve("./"); + const cdirNoPrivate = cdir.replace(/^\/private/, ""); // for macos + // Normalize path to forward slashes to match Bun's resolver output on Windows + const cdirFwd = cdir.replace(/\\/g, "/"); + const cdirPosix = cdirFwd.replace(/^[a-zA-Z]:/, ""); + const filterResolve = new RegExp( + `^(?!\\.\/main\\.ts)(?!${cdirFwd}\/main\\.ts)(?!${cdirPosix}\/main\\.ts)(?!(?:/private)?${cdirNoPrivate}\/wrapper\\.mjs).*\\.ts$` + ); + + let cdirNodeModules = `${cdirFwd}/node_modules/`; + + const filterLoad = new RegExp(`^${cdir}\/main\\.ts$`); + const transpiler = new Bun.Transpiler({ + loader: "ts", + }); + + function replaceRelativeImports(code) { + const imports = transpiler.scanImports(code); + for (const imp of imports) { + if (imp.kind == "import-statement") { + if ( + (imp.path.startsWith(".") || + imp.path.startsWith("/u/") || + imp.path.startsWith("/f/")) && + !imp.path.endsWith(".ts") + ) { + code = code.replaceAll(imp.path, imp.path + ".ts"); + } + } + } + return { + contents: code, + }; + } + + function normalizePath(rawPath) { + return rawPath.split("/").reduce((acc, seg) => { + if (seg === "..") acc.pop(); + else if (seg !== "." && seg !== "") acc.push(seg); + return acc; + }, []).join("/"); + } + + // Resolve a windmill script import path relative to an importer path. + // Bun on Windows may prefix args with "windmill-url:" or strip leading "/". + function resolveWindmillImport(importerPath, importPath) { + const path = importPath.replace(/^windmill-url:/, "").replace(/^\//, ""); + const isAbsolute = path.startsWith("f/") || path.startsWith("u/"); + const endExt = path.endsWith(".ts") ? "" : ".ts"; + const rawScriptPath = isAbsolute + ? `${path}${endExt}` + : `${importerPath}/../${path}${endExt}`; + return { path: normalizePath(rawScriptPath), namespace: "windmill-url" }; + } + + build.onLoad({ filter: filterLoad }, async (args) => { + const code = readFileSync(args.path, "utf8"); + return replaceRelativeImports(code); + }); + + // Load windmill scripts by fetching from the API + build.onLoad({ filter: /.*/, namespace: "windmill-url" }, async (args) => { + const path = args.path.replace(/^windmill-url:/, ""); + const url = `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${path}`; + const req = await fetch(url, { + method: "GET", + headers: { + Authorization: "Bearer " + token, + }, + }); + if (!req.ok) { + throw new Error( + `Failed to find relative import at ${url} (status ${req.status})` + ); + } + const contents = await req.text(); + return { + contents: replaceRelativeImports(contents).contents, + loader: "tsx", + }; + }); + + // Resolve windmill script imports from the file namespace (e.g. from main.ts) + build.onResolve({ filter: filterResolve }, (args) => { + const importerFwd = args.importer?.replace(/\\/g, "/") ?? ""; + if (importerFwd.startsWith(cdirNodeModules)) { + return undefined; + } + const isMainTs = + args.importer == "./main.ts" || importerFwd.endsWith("/main.ts"); + const file_path = isMainTs + ? current_path + : importerFwd.replace(cdirFwd + "/", ""); + return resolveWindmillImport(file_path, args.path); + }); + + // Resolve nested imports from within windmill-url modules + build.onResolve({ filter: /\.ts$/, namespace: "windmill-url" }, (args) => { + const importer = args.importer.replace(/^windmill-url:/, ""); + return resolveWindmillImport(importer, args.path); + }); + }, +}; diff --git a/backend/windmill-worker/nsjail/run.bun.config.proto b/backend/windmill-worker/nsjail/run.bun.config.proto index 8cbee3dec9..afd5c42ba9 100644 --- a/backend/windmill-worker/nsjail/run.bun.config.proto +++ b/backend/windmill-worker/nsjail/run.bun.config.proto @@ -14,6 +14,18 @@ clone_newnet: false clone_newuser: {CLONE_NEWUSER} clone_newcgroup: false +uidmap { + inside_id: "1000" + outside_id: "" + count: 1 +} + +gidmap { + inside_id: "1000" + outside_id: "" + count: 1 +} + skip_setsid: true keep_caps: false keep_env: true @@ -130,6 +142,13 @@ mount { rw: true } +mount { + src: "{JOB_DIR}/checkpoint.json" + dst: "/tmp/{LANG}/checkpoint.json" + is_bind: true + mandatory: false +} + mount { src: "{JOB_DIR}/result.json" dst: "/tmp/{LANG}/result.json" diff --git a/backend/windmill-worker/src/ai/image_handler.rs b/backend/windmill-worker/src/ai/image_handler.rs index 7eb6bb6597..63d8aeaec3 100644 --- a/backend/windmill-worker/src/ai/image_handler.rs +++ b/backend/windmill-worker/src/ai/image_handler.rs @@ -2,8 +2,8 @@ use base64::Engine; use futures; use ulid; use windmill_common::{client::AuthedClient, error::Error}; -use windmill_types::s3::S3Object; use windmill_queue::MiniPulledJob; +use windmill_types::s3::S3Object; use crate::ai::types::*; diff --git a/backend/windmill-worker/src/ai/providers/anthropic.rs b/backend/windmill-worker/src/ai/providers/anthropic.rs index e42e1dc49b..f9f5edf452 100644 --- a/backend/windmill-worker/src/ai/providers/anthropic.rs +++ b/backend/windmill-worker/src/ai/providers/anthropic.rs @@ -1,14 +1,16 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error}; +use windmill_common::{ + ai_google::parse_data_url, ai_providers::AIProvider, client::AuthedClient, error::Error, +}; use crate::ai::{ image_handler::prepare_messages_for_api, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, sse::{AnthropicSSEParser, SSEParser}, types::*, - utils::{extract_text_content, parse_data_url, should_use_structured_output_tool}, + utils::{extract_text_content, should_use_structured_output_tool}, }; /// Anthropic API version for standard API diff --git a/backend/windmill-worker/src/ai/providers/google_ai.rs b/backend/windmill-worker/src/ai/providers/google_ai.rs index 62e6afff75..e2030f3269 100644 --- a/backend/windmill-worker/src/ai/providers/google_ai.rs +++ b/backend/windmill-worker/src/ai/providers/google_ai.rs @@ -1,215 +1,21 @@ use async_trait::async_trait; -use serde::{Deserialize, Serialize}; -use windmill_common::{client::AuthedClient, error::Error}; +use windmill_common::{ + ai_google::{ + openai_messages_to_gemini, openai_tools_to_gemini, GeminiGenerationConfig, + GeminiImageContent, GeminiImageRequest, GeminiImageResponse, GeminiInlineData, GeminiPart, + GeminiPredictContent, GeminiTextRequest, GeminiTool, + }, + client::AuthedClient, + error::Error, +}; use crate::ai::{ - image_handler::download_and_encode_s3_image, + image_handler::{download_and_encode_s3_image, prepare_messages_for_api}, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, sse::{GeminiSSEParser, SSEParser}, types::*, - utils::parse_data_url, }; -// ============================================================================ -// Gemini API Types - Shared between text and image -// ============================================================================ - -/// Inline data for binary content (images) -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct GeminiInlineData { - #[serde(rename = "mimeType")] - pub mime_type: String, - pub data: String, -} - -/// A part of content - can be text, inline data, function call, or function response -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(untagged)] -pub enum GeminiPart { - Text { - text: String, - }, - InlineData { - #[serde(rename = "inlineData")] - inline_data: GeminiInlineData, - }, - FunctionCall { - #[serde(rename = "functionCall")] - function_call: GeminiFunctionCall, - /// Thought signature for Gemini 3+ models - required for function calling - #[serde(rename = "thoughtSignature", skip_serializing_if = "Option::is_none")] - thought_signature: Option, - }, - FunctionResponse { - #[serde(rename = "functionResponse")] - function_response: GeminiFunctionResponse, - }, -} - -/// A function call from the model -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct GeminiFunctionCall { - pub name: String, - pub args: serde_json::Value, -} - -/// A function response to send back to the model -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct GeminiFunctionResponse { - pub name: String, - pub response: serde_json::Value, -} - -// ============================================================================ -// Gemini Text API Request Types -// ============================================================================ - -/// Main request structure for Gemini generateContent -#[derive(Serialize)] -pub struct GeminiTextRequest { - pub contents: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub tools: Option>, - #[serde(rename = "toolConfig", skip_serializing_if = "Option::is_none")] - pub tool_config: Option, - #[serde(rename = "systemInstruction", skip_serializing_if = "Option::is_none")] - pub system_instruction: Option, - #[serde(rename = "generationConfig", skip_serializing_if = "Option::is_none")] - pub generation_config: Option, -} - -/// Content message with role and parts -#[derive(Serialize)] -pub struct GeminiContentMessage { - #[serde(skip_serializing_if = "Option::is_none")] - pub role: Option, - pub parts: Vec, -} - -/// Tool definition - either function declarations or Google Search -#[derive(Serialize)] -pub struct GeminiTool { - #[serde( - rename = "functionDeclarations", - skip_serializing_if = "Option::is_none" - )] - pub function_declarations: Option>, - #[serde(rename = "googleSearch", skip_serializing_if = "Option::is_none")] - pub google_search: Option, -} - -/// Function declaration for tool use -#[derive(Serialize)] -pub struct GeminiFunctionDeclaration { - pub name: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, - pub parameters: OpenAPISchema, -} - -/// Tool configuration for controlling function calling behavior -#[derive(Serialize)] -pub struct GeminiToolConfig { - #[serde(rename = "functionCallingConfig")] - pub function_calling_config: GeminiFunctionCallingConfig, -} - -/// Function calling configuration -#[derive(Serialize)] -pub struct GeminiFunctionCallingConfig { - pub mode: String, - #[serde( - rename = "allowedFunctionNames", - skip_serializing_if = "Option::is_none" - )] - pub allowed_function_names: Option>, -} - -/// Generation configuration for output format -#[derive(Serialize)] -pub struct GeminiGenerationConfig { - #[serde(skip_serializing_if = "Option::is_none")] - pub temperature: Option, - #[serde(rename = "maxOutputTokens", skip_serializing_if = "Option::is_none")] - pub max_output_tokens: Option, - #[serde(rename = "responseMimeType", skip_serializing_if = "Option::is_none")] - pub response_mime_type: Option, - #[serde(rename = "responseSchema", skip_serializing_if = "Option::is_none")] - pub response_schema: Option, -} - -// ============================================================================ -// Gemini API Response Types -// ============================================================================ - -/// Grounding metadata from Google Search -#[derive(Deserialize)] -#[allow(dead_code)] -pub struct GeminiGroundingMetadata { - #[serde(rename = "webSearchQueries")] - pub web_search_queries: Option>, - #[serde(rename = "groundingChunks")] - pub grounding_chunks: Option>, -} - -// ============================================================================ -// Gemini Image API Types (for Imagen models) -// ============================================================================ - -/// Request for image generation (Imagen models) -#[derive(Serialize)] -pub struct GeminiImageRequest { - #[serde(skip_serializing_if = "Option::is_none")] - pub contents: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub instances: Option>, -} - -/// Content for image generation -#[derive(Serialize)] -pub struct GeminiImageContent { - pub parts: Vec, -} - -/// Content for Imagen predict endpoint -#[derive(Serialize)] -pub struct GeminiPredictContent { - pub prompt: String, -} - -/// Response for image generation -#[derive(Deserialize)] -pub struct GeminiImageResponse { - pub candidates: Option>, - pub predictions: Option>, -} - -/// Image candidate from generateContent -#[derive(Deserialize)] -pub struct GeminiImageCandidate { - pub content: GeminiImageCandidateContent, -} - -/// Content in image candidate -#[derive(Deserialize)] -pub struct GeminiImageCandidateContent { - pub parts: Vec, -} - -/// Part of image candidate -#[derive(Deserialize)] -pub struct GeminiImageCandidatePart { - #[serde(rename = "inlineData", skip_serializing_if = "Option::is_none")] - pub inline_data: Option, -} - -/// Prediction candidate from Imagen -#[derive(Deserialize)] -pub struct GeminiPredictCandidate { - #[serde(rename = "bytesBase64Encoded")] - pub bytes_base64_encoded: String, -} - // ============================================================================ // Query Builder Implementation // ============================================================================ @@ -221,34 +27,24 @@ impl GoogleAIQueryBuilder { Self } - /// Build a text request using the native Gemini API format async fn build_text_request( &self, args: &BuildRequestArgs<'_>, client: &AuthedClient, workspace_id: &str, ) -> Result { - // Convert messages to Gemini format - let contents = self - .convert_messages_to_gemini(args.messages, client, workspace_id) - .await?; + let prepared_messages = + prepare_messages_for_api(args.messages, client, workspace_id).await?; + let (contents, system_instruction) = openai_messages_to_gemini(&prepared_messages); - // Build tools array let tools = self.convert_tools_to_gemini(args.tools, args.has_websearch); - // Build generation config let generation_config = self.build_generation_config(args); - // Build system instruction from system_prompt - let system_instruction = args.system_prompt.map(|s| GeminiContentMessage { - role: None, - parts: vec![GeminiPart::Text { text: s.to_string() }], - }); - let request = GeminiTextRequest { contents, tools, - tool_config: None, // Use AUTO mode by default + tool_config: None, system_instruction, generation_config, }; @@ -257,7 +53,6 @@ impl GoogleAIQueryBuilder { .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) } - /// Build an image generation request async fn build_image_request( &self, args: &BuildRequestArgs<'_>, @@ -267,7 +62,6 @@ impl GoogleAIQueryBuilder { let is_imagen = args.model.contains("imagen"); let request = if is_imagen { - // For Imagen models, use simple prompt format GeminiImageRequest { instances: Some(vec![GeminiPredictContent { prompt: args.user_message.trim().to_string(), @@ -275,7 +69,6 @@ impl GoogleAIQueryBuilder { contents: None, } } else { - // For Gemini models with image generation, build parts let mut parts = vec![GeminiPart::Text { text: args.user_message.trim().to_string() }]; if let Some(system_prompt) = args.system_prompt { @@ -285,7 +78,6 @@ impl GoogleAIQueryBuilder { ); } - // Add input images if provided if let Some(images) = args.images { for image in images.iter() { if !image.s3.is_empty() { @@ -308,218 +100,39 @@ impl GoogleAIQueryBuilder { .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) } - /// Convert OpenAI-format messages to Gemini format - async fn convert_messages_to_gemini( - &self, - messages: &[OpenAIMessage], - client: &AuthedClient, - workspace_id: &str, - ) -> Result, Error> { - let mut gemini_messages = Vec::new(); - - for msg in messages { - match msg.role.as_str() { - "system" => { - // Skip - handled via args.system_prompt in build_text_request - } - "tool" => { - // Handle tool responses - if let (Some(tool_call_id), Some(content)) = (&msg.tool_call_id, &msg.content) { - let func_name = self.find_function_name_by_id(messages, tool_call_id); - let response_text = match content { - OpenAIContent::Text(text) => text.clone(), - OpenAIContent::Parts(parts) => parts - .iter() - .filter_map(|p| match p { - ContentPart::Text { text } => Some(text.clone()), - _ => None, - }) - .collect::>() - .join(" "), - }; - - gemini_messages.push(GeminiContentMessage { - role: Some("user".to_string()), - parts: vec![GeminiPart::FunctionResponse { - function_response: GeminiFunctionResponse { - name: func_name, - response: serde_json::json!({ "result": response_text }), - }, - }], - }); - } - } - _ => { - // Handle user/assistant messages - let role = match msg.role.as_str() { - "assistant" => "model", - _ => "user", - }; - - let mut parts = Vec::new(); - - // Handle regular content - if let Some(content) = &msg.content { - let content_parts = self - .convert_content_to_parts(&Some(content.clone()), client, workspace_id) - .await?; - parts.extend(content_parts); - } - - // Handle tool calls from assistant - if let Some(tool_calls) = &msg.tool_calls { - for tc in tool_calls { - let args: serde_json::Value = - serde_json::from_str(&tc.function.arguments).unwrap_or_default(); - // Extract thought_signature from extra_content if present - let thought_signature = tc - .extra_content - .as_ref() - .and_then(|ec| ec.google.as_ref()) - .and_then(|g| g.thought_signature.clone()); - parts.push(GeminiPart::FunctionCall { - function_call: GeminiFunctionCall { - name: tc.function.name.clone(), - args, - }, - thought_signature, - }); - } - } - - if !parts.is_empty() { - gemini_messages - .push(GeminiContentMessage { role: Some(role.to_string()), parts }); - } - } - } - } - - Ok(gemini_messages) - } - - /// Convert OpenAI content to Gemini parts - async fn convert_content_to_parts( - &self, - content: &Option, - client: &AuthedClient, - workspace_id: &str, - ) -> Result, Error> { - let mut parts = Vec::new(); - - if let Some(content) = content { - match content { - OpenAIContent::Text(text) => { - if !text.is_empty() { - parts.push(GeminiPart::Text { text: text.clone() }); - } - } - OpenAIContent::Parts(content_parts) => { - for part in content_parts { - match part { - ContentPart::Text { text } => { - if !text.is_empty() { - parts.push(GeminiPart::Text { text: text.clone() }); - } - } - ContentPart::ImageUrl { image_url } => { - // Parse data URL format: data:mime_type;base64,data - if let Some((mime_type, data)) = parse_data_url(&image_url.url) { - parts.push(GeminiPart::InlineData { - inline_data: GeminiInlineData { mime_type, data }, - }); - } - } - ContentPart::S3Object { s3_object } => { - if !s3_object.s3.is_empty() { - let (mime_type, data) = download_and_encode_s3_image( - s3_object, - client, - workspace_id, - ) - .await?; - parts.push(GeminiPart::InlineData { - inline_data: GeminiInlineData { mime_type, data }, - }); - } - } - } - } - } - } - } - - Ok(parts) - } - - /// Find function name by tool call ID from previous messages - fn find_function_name_by_id(&self, messages: &[OpenAIMessage], tool_call_id: &str) -> String { - for msg in messages { - if let Some(tool_calls) = &msg.tool_calls { - for tc in tool_calls { - if tc.id == tool_call_id { - return tc.function.name.clone(); - } - } - } - } - "unknown_function".to_string() - } - - /// Convert OpenAI tools to Gemini format + /// Convert OpenAI tool definitions to Gemini format. + /// + /// Sanitizes each tool's JSON schema for Google compatibility before delegating + /// to the shared [`openai_tools_to_gemini`] function. fn convert_tools_to_gemini( &self, tools: Option<&[ToolDef]>, has_websearch: bool, ) -> Option> { - let mut gemini_tools = Vec::new(); - - // Add function declarations - if let Some(tool_defs) = tools { - let declarations: Vec = tool_defs - .iter() - .filter_map(|t| { - // Deserialize RawValue into OpenAPISchema, sanitize, then use - let mut schema: OpenAPISchema = - serde_json::from_str(t.function.parameters.get()).ok()?; - schema.sanitize_for_google(); - - Some(GeminiFunctionDeclaration { - name: t.function.name.clone(), - description: t.function.description.clone(), - parameters: schema, - }) - }) - .collect(); - - if !declarations.is_empty() { - gemini_tools.push(GeminiTool { - function_declarations: Some(declarations), - google_search: None, - }); + let Some(tool_defs) = tools else { + if has_websearch { + return Some(vec![GeminiTool { + function_declarations: None, + google_search: Some(serde_json::json!({})), + }]); } - } + return None; + }; - // Add Google Search tool if enabled - if has_websearch { - gemini_tools.push(GeminiTool { - function_declarations: None, - google_search: Some(serde_json::json!({})), - }); - } + let tool_params: Vec = tool_defs + .iter() + .map(|t| { + let mut schema: OpenAPISchema = + serde_json::from_str(t.function.parameters.get()).unwrap_or_default(); + schema.sanitize_for_google(); + serde_json::to_value(&schema).unwrap_or_default() + }) + .collect(); - if gemini_tools.is_empty() { - None - } else { - Some(gemini_tools) - } + openai_tools_to_gemini(tool_defs, &tool_params, has_websearch) } - /// Build generation config for structured output and other settings - fn build_generation_config( - &self, - args: &BuildRequestArgs<'_>, - ) -> Option { + fn build_generation_config(&self, args: &BuildRequestArgs<'_>) -> Option { let has_output_schema = args .output_schema .and_then(|s| s.properties.as_ref()) @@ -529,15 +142,11 @@ impl GoogleAIQueryBuilder { let (response_mime_type, response_schema) = if has_output_schema { let mut schema = args.output_schema.unwrap().clone(); schema.sanitize_for_google(); - ( - Some("application/json".to_string()), - serde_json::to_value(&schema).ok(), - ) + (Some("application/json".to_string()), serde_json::to_value(&schema).ok()) } else { (None, None) }; - // Only create config if there's something to configure if args.temperature.is_some() || args.max_tokens.is_some() || response_mime_type.is_some() { Some(GeminiGenerationConfig { temperature: args.temperature, @@ -554,7 +163,6 @@ impl GoogleAIQueryBuilder { #[async_trait] impl QueryBuilder for GoogleAIQueryBuilder { fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool { - // Google AI supports tools only for text output matches!(output_type, OutputType::Text) } @@ -578,7 +186,6 @@ impl QueryBuilder for GoogleAIQueryBuilder { Error::internal_err(format!("Failed to parse Gemini image response: {}", e)) })?; - // First, check Gemini models (candidates -> content -> parts -> inline_data) let image_data_from_gemini = gemini_response.candidates.as_ref().and_then(|candidates| { candidates.iter().find_map(|candidate| { candidate @@ -589,13 +196,11 @@ impl QueryBuilder for GoogleAIQueryBuilder { }) }); - // Then, check Imagen models (predictions -> bytes_base64_encoded) let image_data_from_imagen = gemini_response .predictions .as_ref() .and_then(|predictions| predictions.first().map(|p| &p.bytes_base64_encoded)); - // Image data, preferring Gemini first then Imagen models let image_data = image_data_from_gemini.or(image_data_from_imagen); match image_data { @@ -627,7 +232,6 @@ impl QueryBuilder for GoogleAIQueryBuilder { .. } = gemini_sse_parser; - // Send tool call arguments events for accumulated tool calls for tool_call in accumulated_tool_calls.values() { let event = StreamingEvent::ToolCallArguments { call_id: tool_call.id.clone(), @@ -637,7 +241,6 @@ impl QueryBuilder for GoogleAIQueryBuilder { stream_event_processor.send(event, &mut events_str).await?; } - // Convert Gemini usage metadata to TokenUsage let usage = gemini_usage.map(|u| { TokenUsage::new( u.prompt_token_count, @@ -647,11 +250,7 @@ impl QueryBuilder for GoogleAIQueryBuilder { }); Ok(ParsedResponse::Text { - content: if accumulated_content.is_empty() { - None - } else { - Some(accumulated_content) - }, + content: if accumulated_content.is_empty() { None } else { Some(accumulated_content) }, tool_calls: accumulated_tool_calls.into_values().collect(), events_str: Some(events_str), annotations, @@ -663,17 +262,11 @@ impl QueryBuilder for GoogleAIQueryBuilder { fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String { match output_type { OutputType::Text => { - format!( - "{}/models/{}:streamGenerateContent?alt=sse", - base_url, model - ) + format!("{}/models/{}:streamGenerateContent?alt=sse", base_url, model) } OutputType::Image => { - let url_suffix = if model.contains("imagen") { - "predict" - } else { - "generateContent" - }; + let url_suffix = + if model.contains("imagen") { "predict" } else { "generateContent" }; format!("{}/models/{}:{}", base_url, model, url_suffix) } } @@ -685,7 +278,6 @@ impl QueryBuilder for GoogleAIQueryBuilder { _base_url: &str, _output_type: &OutputType, ) -> Vec<(&'static str, String)> { - // Native Gemini API always uses x-goog-api-key vec![("x-goog-api-key", api_key.to_string())] } } diff --git a/backend/windmill-worker/src/ai/query_builder.rs b/backend/windmill-worker/src/ai/query_builder.rs index 04f1b4b548..73010b5ba1 100644 --- a/backend/windmill-worker/src/ai/query_builder.rs +++ b/backend/windmill-worker/src/ai/query_builder.rs @@ -1,7 +1,5 @@ use async_trait::async_trait; -use windmill_common::{ - client::AuthedClient, error::Error, worker::Connection, -}; +use windmill_common::{client::AuthedClient, error::Error, worker::Connection}; use windmill_queue::MiniPulledJob; use windmill_types::s3::S3Object; diff --git a/backend/windmill-worker/src/ai/sse.rs b/backend/windmill-worker/src/ai/sse.rs index 77fba3140a..62f13f3494 100644 --- a/backend/windmill-worker/src/ai/sse.rs +++ b/backend/windmill-worker/src/ai/sse.rs @@ -5,15 +5,18 @@ use reqwest::Response; use serde::Deserialize; use serde_json; use tokio_stream::StreamExt; -use windmill_common::{error::Error, utils::rd_string}; +use windmill_common::{ + ai_google::{parse_gemini_sse_event, GeminiUsageMetadata}, + ai_types::{ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall}, + error::Error, + utils::rd_string, +}; use crate::ai::{ query_builder::StreamEventProcessor, types::{StreamingEvent, UrlCitation}, }; -use windmill_common::ai_types::{ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall}; - #[derive(Deserialize)] pub struct OpenAIChoiceDeltaToolCallFunction { pub name: Option, @@ -457,96 +460,19 @@ impl SSEParser for AnthropicSSEParser { // Gemini SSE Parser // ============================================================================ -/// Gemini streaming response part - can be text or function call -#[derive(Deserialize, Debug)] -pub struct GeminiSSEPart { - #[serde(default)] - pub text: Option, - #[serde(rename = "functionCall")] - pub function_call: Option, - /// Thought signature for Gemini 3+ models - required for function calling - #[serde(rename = "thoughtSignature")] - pub thought_signature: Option, -} - -/// Function call in Gemini streaming response -#[derive(Deserialize, Debug)] -pub struct GeminiSSEFunctionCall { - pub name: String, - pub args: serde_json::Value, -} - -/// Content in Gemini streaming candidate -#[derive(Deserialize, Debug)] -pub struct GeminiSSEContent { - pub parts: Option>, -} - -/// Web reference in Gemini grounding chunk -#[derive(Deserialize, Debug)] -pub struct GeminiGroundingChunkWeb { - pub uri: String, - #[serde(default)] - pub title: Option, -} - -/// Grounding chunk from Gemini web search -#[derive(Deserialize, Debug)] -pub struct GeminiGroundingChunk { - pub web: Option, -} - -/// Grounding metadata from Gemini web search -#[derive(Deserialize, Debug)] -pub struct GeminiGroundingMetadata { - #[serde(rename = "groundingChunks", default)] - pub grounding_chunks: Vec, - #[serde(rename = "webSearchQueries", default)] - pub web_search_queries: Vec, -} - -/// Candidate in Gemini streaming response -#[derive(Deserialize, Debug)] -pub struct GeminiSSECandidate { - pub content: Option, - #[serde(rename = "finishReason")] - #[allow(dead_code)] - pub finish_reason: Option, - #[serde(rename = "groundingMetadata")] - pub grounding_metadata: Option, -} - -/// Gemini usage metadata from SSE response -#[derive(Deserialize, Debug, Clone)] -pub struct GeminiUsageMetadata { - #[serde(rename = "promptTokenCount", default)] - pub prompt_token_count: Option, - #[serde(rename = "candidatesTokenCount", default)] - pub candidates_token_count: Option, - #[serde(rename = "totalTokenCount", default)] - pub total_token_count: Option, -} - -/// Gemini SSE event structure -#[derive(Deserialize, Debug)] -pub struct GeminiSSEEvent { - pub candidates: Option>, - #[serde(rename = "usageMetadata")] - pub usage_metadata: Option, -} - -/// Gemini SSE Parser for streaming responses +/// Accumulates Gemini streaming events and converts them into the worker's +/// internal [`OpenAIToolCall`] / [`StreamingEvent`] representation. +/// +/// The actual SSE parsing is delegated to [`parse_gemini_sse_event`] from +/// `windmill_common::ai_google` so the logic can be shared with the API proxy. pub struct GeminiSSEParser { pub accumulated_content: String, pub accumulated_tool_calls: HashMap, pub events_str: String, pub stream_event_processor: StreamEventProcessor, tool_call_index: i64, - /// Collected URL citation annotations from web search pub annotations: Vec, - /// Whether web search was used in this response pub used_websearch: bool, - /// Token usage from usageMetadata pub usage: Option, } @@ -567,101 +493,57 @@ impl GeminiSSEParser { impl SSEParser for GeminiSSEParser { async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { - let event: Option = serde_json::from_str(data) - .inspect_err(|e| { - tracing::error!("Failed to parse SSE as a Gemini event {}: {}", data, e); - }) - .ok(); + let Some(parsed) = parse_gemini_sse_event(data)? else { + return Ok(()); + }; - if let Some(event) = event { - if let Some(candidates) = event.candidates { - for candidate in candidates { - if let Some(content) = candidate.content { - if let Some(parts) = content.parts { - for part in parts { - // Handle text content - if let Some(text) = part.text { - if !text.is_empty() { - self.accumulated_content.push_str(&text); - let event = StreamingEvent::TokenDelta { content: text }; - self.stream_event_processor - .send(event, &mut self.events_str) - .await?; - } - } + if let Some(text) = parsed.text { + self.accumulated_content.push_str(&text); + self.stream_event_processor + .send(StreamingEvent::TokenDelta { content: text }, &mut self.events_str) + .await?; + } - // Handle function calls - if let Some(function_call) = part.function_call { - let call_id = format!("call_{}", rd_string(24)); - let idx = self.tool_call_index; - self.tool_call_index += 1; + for tool_call in parsed.tool_calls { + let call_id = format!("call_{}", rd_string(24)); + let idx = self.tool_call_index; + self.tool_call_index += 1; - // Send tool call start event - let event = StreamingEvent::ToolCall { - call_id: call_id.clone(), - function_name: function_call.name.clone(), - }; - self.stream_event_processor - .send(event, &mut self.events_str) - .await?; + self.stream_event_processor + .send( + StreamingEvent::ToolCall { + call_id: call_id.clone(), + function_name: tool_call.name.clone(), + }, + &mut self.events_str, + ) + .await?; - // Build extra_content with thought_signature if present - let extra_content = - part.thought_signature.map(|sig| ExtraContent { - google: Some(GoogleExtraContent { - thought_signature: Some(sig), - }), - }); + let extra_content = tool_call.thought_signature.map(|sig| ExtraContent { + google: Some(GoogleExtraContent { thought_signature: Some(sig) }), + }); - // Store accumulated tool call - self.accumulated_tool_calls.insert( - idx, - OpenAIToolCall { - id: call_id, - function: OpenAIFunction { - name: function_call.name, - arguments: serde_json::to_string( - &function_call.args, - ) - .unwrap_or_else(|_| "{}".to_string()), - }, - r#type: "function".to_string(), - extra_content, - }, - ); - } - } - } - } + self.accumulated_tool_calls.insert( + idx, + OpenAIToolCall { + id: call_id, + function: OpenAIFunction { + name: tool_call.name, + arguments: serde_json::to_string(&tool_call.args) + .unwrap_or_else(|_| "{}".to_string()), + }, + r#type: "function".to_string(), + extra_content, + }, + ); + } - // Handle grounding metadata (web search results) - if let Some(ref grounding_metadata) = candidate.grounding_metadata { - // Set used_websearch if there are search queries or grounding chunks - if !grounding_metadata.web_search_queries.is_empty() - || !grounding_metadata.grounding_chunks.is_empty() - { - self.used_websearch = true; - } - - // Extract citations from grounding chunks - for chunk in &grounding_metadata.grounding_chunks { - if let Some(ref web) = chunk.web { - self.annotations.push(UrlCitation { - start_index: 0, // Gemini doesn't provide character indices - end_index: 0, - url: web.uri.clone(), - title: web.title.clone(), - }); - } - } - } - } - } - - // Extract usage metadata - if let Some(usage_metadata) = event.usage_metadata { - self.usage = Some(usage_metadata); - } + self.annotations.extend(parsed.annotations); + if parsed.used_websearch { + self.used_websearch = true; + } + if let Some(usage) = parsed.usage { + self.usage = Some(usage); } Ok(()) diff --git a/backend/windmill-worker/src/ai/types.rs b/backend/windmill-worker/src/ai/types.rs index eebe09bc77..631a8ab113 100644 --- a/backend/windmill-worker/src/ai/types.rs +++ b/backend/windmill-worker/src/ai/types.rs @@ -20,8 +20,8 @@ use windmill_common::{ flow_status::AgentAction, flows::FlowModule, }; -use windmill_types::s3::S3Object; use windmill_parser::Typ; +use windmill_types::s3::S3Object; // Re-export shared types from windmill_common::ai_types pub use windmill_common::ai_types::{ @@ -1603,10 +1603,7 @@ mod tests { schema.sanitize_for_google(); - assert!( - schema.multiple_of.is_none(), - "multipleOf should be removed" - ); + assert!(schema.multiple_of.is_none(), "multipleOf should be removed"); } #[test] @@ -1639,7 +1636,10 @@ mod tests { assert!(schema.default.is_none()); let value_prop = schema.properties.as_ref().unwrap().get("value").unwrap(); - assert!(value_prop.default.is_none(), "nested default should be removed"); + assert!( + value_prop.default.is_none(), + "nested default should be removed" + ); assert!( value_prop.exclusive_minimum.is_none(), "nested exclusiveMinimum should be removed" @@ -1652,7 +1652,10 @@ mod tests { value_prop.multiple_of.is_none(), "nested multipleOf should be removed" ); - assert!(value_prop.r#const.is_none(), "nested const should be removed"); + assert!( + value_prop.r#const.is_none(), + "nested const should be removed" + ); assert!(schema.properties.is_some()); assert!(matches!(&schema.r#type, Some(SchemaType::Single(t)) if t == "object")); diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index a374db29c5..d095602b40 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -731,16 +731,3 @@ pub fn extract_text_content(content: &OpenAIContent) -> String { .join(""), } } - -/// Parse a data URL to extract media type and base64 data -/// Format: data:mime_type;base64,data -/// Returns (media_type, data) tuple if successful -pub fn parse_data_url(url: &str) -> Option<(String, String)> { - if !url.starts_with("data:") { - return None; - } - let rest = url.strip_prefix("data:")?; - let (header, data) = rest.split_once(",")?; - let media_type = header.strip_suffix(";base64")?; - Some((media_type.to_string(), data.to_string())) -} diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 30348895cf..7d92607af5 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -26,7 +26,12 @@ use windmill_queue::{ }; lazy_static::lazy_static! { - pub static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| "/bin/bash".to_string()); + pub static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| { + #[cfg(not(windows))] + { "/bin/bash".to_string() } + #[cfg(windows)] + { "bash".to_string() } + }); } const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto"); @@ -210,7 +215,10 @@ exit $exit_status .current_dir(job_dir) .env_clear() .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Bash).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .args(cmd_args) @@ -236,7 +244,10 @@ exit $exit_status .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Bash).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index 2f748fe22c..b092e135b6 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -6,9 +6,9 @@ use reqwest::Client; use serde_json::{json, value::RawValue, Value}; use windmill_common::client::AuthedClient; use windmill_common::error::to_anyhow; -use windmill_object_store::convert_json_line_stream; use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; use windmill_common::{error::Error, worker::to_raw_value}; +use windmill_object_store::convert_json_line_stream; use windmill_parser_sql::{ parse_bigquery_sig, parse_db_resource, parse_s3_mode, parse_sql_blocks, parse_sql_statement_named_params, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 08a61645ce..16900dad8b 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -22,7 +22,7 @@ use crate::{ handle_child::handle_child, is_sandboxing_enabled, read_ee_registry, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_NO_CACHE, BUN_PATH, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, - NPMRC, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + NPMRC, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; use windmill_common::{ @@ -53,7 +53,14 @@ use windmill_object_store::attempt_fetch_bytes; use windmill_parser::Typ; +// The Windows loader uses a virtual "windmill-url" namespace instead of writing .url +// files to disk, which avoids Windows path issues. The virtual namespace approach is +// likely better on all fronts but we keep the original .url-file loader on Linux to +// avoid breaking back-compat. +#[cfg(not(windows))] pub const RELATIVE_BUN_LOADER: &str = include_str!("../loader.bun.js"); +#[cfg(windows)] +pub const RELATIVE_BUN_LOADER: &str = include_str!("../loader.bun.windows.js"); pub const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.js"); @@ -527,6 +534,8 @@ pub async fn build_loader( current_path: &str, mode: LoaderMode, ) -> Result<()> { + // Use forward slashes in JS strings to avoid backslash escape issues on Windows + let job_dir_js = job_dir.replace('\\', "/"); let loader = RELATIVE_BUN_LOADER .replace("W_ID", w_id) .replace("BASE_INTERNAL_URL", base_internal_url) @@ -549,13 +558,13 @@ import {{ readdir }} from "node:fs/promises"; let fileNames = [] try {{ - fileNames = await readdir("{job_dir}/node_modules") + fileNames = await readdir("{job_dir_js}/node_modules") }} catch (e) {{ }} try {{ await Bun.build({{ - entrypoints: ["{job_dir}/wrapper.mjs"], + entrypoints: ["{job_dir_js}/wrapper.mjs"], outdir: "./", target: "node", plugins: [p], @@ -597,7 +606,7 @@ plugin(p) try {{ await Bun.build({{ - entrypoints: ["{job_dir}/main.ts"], + entrypoints: ["{job_dir_js}/main.ts"], outdir: "./", target: "{}", plugins: [p], @@ -981,6 +990,14 @@ pub async fn handle_bun_job( ) -> error::Result> { let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); + if annotation.sandbox && NSJAIL_AVAILABLE.is_none() { + return Err(error::Error::ExecutionErr( + "Script has //sandbox annotation but nsjail is not available on this worker. \ + Please ensure nsjail is installed or remove the //sandbox annotation." + .to_string(), + )); + } + let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if let (Some(lock), true) = ( maybe_lock.get_lock(), !annotation.nobundling && !*DISABLE_BUNDLING && codebase.is_none(), @@ -1019,6 +1036,12 @@ pub async fn handle_bun_job( let _ = write_file(job_dir, "main.ts", inner_content)?; } else if !annotation.native && codebase.is_none() { let _ = write_file(job_dir, "package.json", r#"{ "type": "module" }"#)?; + } else if codebase.is_some() { + // Write a valid fallback package.json for codebase mode. Without this, + // nsjail creates an empty 0-byte file (from the mandatory: false mount) + // which Node.js fails to parse as JSON (ERR_INVALID_PACKAGE_CONFIG). + // If the codebase TAR includes a package.json, it will overwrite this. + let _ = write_file(job_dir, "package.json", "{}")?; }; let common_bun_proc_envs: HashMap = @@ -1028,6 +1051,37 @@ pub async fn handle_bun_job( let apply_preprocessor = job.flow_step_id.as_deref() != Some("preprocessor") && job.preprocessed == Some(false); + let is_wac_v2 = main_override.is_none() && crate::wac_executor::is_wac_v2_ts(inner_content); + + // For WAC v2, inject variable names into unnamed task() calls so the + // runtime can use them for step naming (timeline, graph). + // `const double = task(async ...` → `const double = task("double", async ...` + // Also handles: export const, let, var, and optional generic type parameters. + // Skips calls that already have a string argument: `task("path", async ...` + let inner_content = if is_wac_v2 { + use regex::Regex; + use std::borrow::Cow; + lazy_static::lazy_static! { + static ref TASK_RE: Regex = + Regex::new(r#"(?m)((?:export\s+)?(?:const|let|var)\s+)(\w+)(\s*=\s*task\s*(?:<[^>]*>)?\s*\(\s*)(async\b)"#).unwrap(); + } + let replaced = TASK_RE.replace_all(inner_content, r#"${1}${2}${3}"${2}", ${4}"#); + match replaced { + Cow::Borrowed(_) => inner_content.to_string(), + Cow::Owned(s) => s, + } + } else { + inner_content.to_string() + }; + let inner_content = inner_content.as_str(); + + // WAC v2 scripts can't use bundle caching because the wrapper imports + // windmill-client from node_modules, which isn't available in bundle mode + if is_wac_v2 && has_bundle_cache { + has_bundle_cache = false; + let _ = write_file(job_dir, "main.ts", inner_content)?; + } + let mut format = BundleFormat::Cjs; if has_bundle_cache { let target; @@ -1152,18 +1206,29 @@ pub async fn handle_bun_job( init_logs = format!("\n{}{}", cache_logs, init_logs); } + if annotation.sandbox { + init_logs.push_str("sandbox mode (nsjail)\n"); + } + let write_wrapper_f = async { if !has_bundle_cache && annotation.native { return Ok(()) as error::Result<()>; } // let mut start = Instant::now(); - let args = windmill_parser_ts::parse_deno_signature( - inner_content, - true, - false, - main_override.map(ToString::to_string), - )? - .args; + let args = if is_wac_v2 { + // For WAC v2, try to parse "main" args; if that fails, try the default export + windmill_parser_ts::parse_deno_signature(inner_content, true, false, None) + .unwrap_or_default() + .args + } else { + windmill_parser_ts::parse_deno_signature( + inner_content, + true, + false, + main_override.map(ToString::to_string), + )? + .args + }; let pre_args = if apply_preprocessor { Some( @@ -1200,6 +1265,14 @@ pub async fn handle_bun_job( // we cannot use Bun.read and Bun.write because it results in an EBADF error on cloud let main_name = main_override.unwrap_or("main"); + // For WAC child jobs where the parser can't find params (task-wrapped consts), + // fall back to passing arg values directly (filtering out internal fields) + let child_spread = if spread.is_empty() && main_override.is_some() { + "Object.values(Object.fromEntries(Object.entries(args).filter(([k]) => !k.startsWith('_'))))".to_string() + } else { + "argsObjToArr(args)".to_string() + }; + let main_import = if codebase.is_some() || has_bundle_cache { "./main.js" } else { @@ -1223,8 +1296,116 @@ pub async fn handle_bun_job( "".to_string() }; - let wrapper_content = format!( - r#" + let wac_spread = if spread.is_empty() { + "Object.values(args)".to_string() + } else { + format!("argsObjToArr(args)") + }; + + let wrapper_content = if is_wac_v2 { + format!( + r#" +import * as Main from "{main_import}"; +import {{ WorkflowCtx, StepSuspend, setWorkflowCtx }} from "windmill-client"; + +import * as fs from "fs/promises"; + +let args = await fs.readFile('args.json', {{ encoding: 'utf8' }}).then(JSON.parse); +const checkpoint = JSON.parse(await fs.readFile('checkpoint.json', {{ encoding: 'utf8' }})); + +function argsObjToArr({{ {spread} }}) {{ + return [ {spread} ]; +}} + +BigInt.prototype.toJSON = function () {{ + return this.toString(); +}}; + +// Find the workflow entrypoint (export default) +let workflowFn = Main.default; +if (!workflowFn || !workflowFn._is_workflow) {{ + for (const key of Object.keys(Main)) {{ + if (Main[key]?._is_workflow) {{ + workflowFn = Main[key]; + break; + }} + }} +}} +if (!workflowFn) {{ + throw new Error("No workflow() entrypoint found. Wrap your main function with workflow()."); +}} + +async function run() {{ + {dates} + {preprocessor} + const argsArr = {wac_spread}; + + const ctx = new WorkflowCtx(checkpoint); + setWorkflowCtx(ctx); + + try {{ + const result = await workflowFn(...argsArr); + setWorkflowCtx(null); + // Flush any unawaited tasks (e.g. forgotten await on last statement) + const trailing = ctx._flushPending(); + if (trailing.length > 0) {{ + return {{ type: "dispatch", mode: trailing.length > 1 ? "parallel" : "sequential", steps: trailing }}; + }} + return {{ type: "complete", result: result ?? null }}; + }} catch (e) {{ + setWorkflowCtx(null); + if (e?.name === "StepSuspend" || e instanceof StepSuspend) {{ + const dispatch = e.dispatchInfo ?? e.dispatch_info ?? {{}}; + if (dispatch.mode === "step_complete") {{ + return {{ type: "complete", result: dispatch.result ?? null }}; + }} + if (dispatch.mode === "inline_checkpoint") {{ + return {{ type: "inline_checkpoint", key: dispatch.key, result: dispatch.result ?? null }}; + }} + if (dispatch.mode === "approval") {{ + return {{ type: "approval", key: dispatch.key, timeout: dispatch.timeout, form: dispatch.form }}; + }} + if (dispatch.mode === "sleep") {{ + return {{ type: "sleep", key: dispatch.key, seconds: dispatch.seconds }}; + }} + return {{ type: "dispatch", mode: dispatch.mode ?? "sequential", steps: dispatch.steps ?? [] }}; + }} + throw e; + }} +}} + +try {{ + const output = await run(); + const output_json = JSON.stringify(output, (key, value) => + typeof value === 'undefined' ? null : value + ); + await fs.writeFile("result.json", output_json); + process.exit(0); +}} catch(e) {{ + console.error(e); + let err = {{ message: e.message, name: e.name, stack: e.stack }}; + let step_id = process.env.WM_FLOW_STEP_ID; + if (step_id) {{ + err["step_id"] = step_id; + }} + const extra = {{}}; + Object.getOwnPropertyNames(e).forEach((key) => {{ + if (['line', 'name', 'stack', 'column', 'message', 'sourceURL', 'originalLine', 'originalColumn'].includes(key)) {{ + return; + }} + extra[key] = e[key]; + }}); + if (Object.keys(extra).length > 0) {{ + err["extra"] = extra; + }} + await fs.writeFile("result.json", JSON.stringify(err)); + process.exit(1); +}} + "#, + ) + } else { + format!( + r#" import * as Main from "{main_import}"; import * as fs from "fs/promises"; @@ -1246,11 +1427,14 @@ BigInt.prototype.toJSON = function () {{ async function run() {{ {dates} {preprocessor} - const argsArr = argsObjToArr(args); + // If the entrypoint has no parsed params (spread is empty), pass values directly + // This handles WAC child jobs where tasks are const-wrapped functions + const argsArr = {child_spread}; if (Main.{main_name} === undefined || typeof Main.{main_name} !== 'function') {{ throw new Error("{main_name} function is missing"); }} - let res = await Main.{main_name}(...argsArr); + let entrypoint = Main.{main_name}; + let res = await entrypoint(...argsArr); if (isAsyncIterable(res)) {{ for await (const chunk of res) {{ console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n')); @@ -1284,7 +1468,8 @@ try {{ process.exit(1); }} "#, - ); + ) + }; write_file(job_dir, "wrapper.mjs", &wrapper_content)?; Ok(()) as error::Result<()> }; @@ -1309,6 +1494,7 @@ try {{ && !annotation.nobundling && !*DISABLE_BUNDLING && !codebase.is_some() + && !is_wac_v2 && (maybe_lock.get_lock().is_some() || annotation.native); let write_loader_f = async { @@ -1354,6 +1540,23 @@ try {{ write_wrapper_f, write_loader_f )?; + + // For WAC v2, write checkpoint.json before bun runs + if is_wac_v2 { + if let Connection::Sql(db) = conn { + let checkpoint = crate::wac_executor::load_checkpoint(db, &job.id).await?; + let checkpoint = + crate::wac_executor::prepare_checkpoint_for_resume(db, &job.id, checkpoint).await?; + + let checkpoint_json = serde_json::to_string(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + write_file(job_dir, "checkpoint.json", &checkpoint_json)?; + } else { + write_file(job_dir, "checkpoint.json", r#"{"completed_steps":{}}"#)?; + } + } + if !codebase.is_some() && !has_bundle_cache { if build_cache { generate_bun_bundle( @@ -1449,7 +1652,7 @@ try {{ let result = crate::js_eval::eval_fetch_timeout( env_code, - inner_content.clone(), + inner_content.to_string(), js_code, job_args, job.script_entrypoint_override.clone(), @@ -1476,7 +1679,7 @@ try {{ append_logs(&job.id, &job.workspace_id, init_logs, conn).await; //do not cache local dependencies - let child = if is_sandboxing_enabled() { + let child = if is_sandboxing_enabled() || annotation.sandbox { let _ = write_file( job_dir, "run.config.proto", @@ -1537,7 +1740,9 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Bun).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn).await?, + ) .envs(common_bun_proc_envs) .env("PATH", PATH_ENV.as_str()) .args(args) @@ -1555,7 +1760,10 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Bun).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn) + .await?, + ) .envs(common_bun_proc_envs) .stdin(Stdio::null()) .stdout(Stdio::piped()) @@ -1586,7 +1794,10 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Bun).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn) + .await?, + ) .envs(common_bun_proc_envs) .stdin(Stdio::null()) .stdout(Stdio::piped()) @@ -1644,7 +1855,766 @@ try {{ })?; *new_args = Some(args.clone()); } - read_result(job_dir, handle_result.result_stream).await + + let result = read_result(job_dir, handle_result.result_stream).await?; + + // WAC v2 post-execution: parse output and handle dispatch/suspend + if is_wac_v2 { + return handle_wac_v2_output(result, job, conn).await; + } + + Ok(result) +} + +/// Handle WAC v2 output after bun/python exits. Parse result as WacOutput, +/// dispatch child jobs on suspend, or return the final result. +pub async fn handle_wac_v2_output( + result: Box, + job: &MiniPulledJob, + conn: &Connection, +) -> error::Result> { + use crate::wac_executor::{ + add_completed_step, load_checkpoint, parse_wac_output, update_checkpoint_for_dispatch, + WacOutput, + }; + use serde_json::Value; + use windmill_common::get_latest_flow_version_info_for_path; + use windmill_common::jobs::{script_path_to_payload, JobKind, JobPayload, RawCode}; + use windmill_common::runnable_settings::{ + ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, + }; + use windmill_queue::{push, PushArgs, PushIsolationLevel}; + + let output = parse_wac_output(&result)?; + + match output { + WacOutput::Complete { result: value } => { + // Workflow completed — return the inner result value + let raw = serde_json::value::to_raw_value(&value).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize WAC result: {e}")) + })?; + Ok(raw) + } + WacOutput::Dispatch { mode, steps } => { + if steps.is_empty() { + return Err(error::Error::internal_err( + "WAC v2 dispatch with no steps — this is a bug in the workflow SDK".to_string(), + )); + } + let db = match conn { + Connection::Sql(db) => db, + _ => { + return Err(error::Error::internal_err( + "WAC v2 dispatch requires SQL connection".to_string(), + )) + } + }; + + let mut checkpoint = load_checkpoint(db, &job.id).await?; + + // Source hash validation: detect if code changed between replays + let current_hash = job.runnable_id.map(|h| h.0.to_string()).unwrap_or_default(); + if !current_hash.is_empty() { + if checkpoint.source_hash.is_empty() { + checkpoint.source_hash = current_hash.clone(); + } else if checkpoint.source_hash != current_hash { + return Err(error::Error::ExecutionErr( + "Workflow source code changed between replays. \ + Cannot safely resume from checkpoint — step keys may have shifted. \ + Please restart this workflow." + .to_string(), + )); + } + } + let num_steps = steps.len(); + + tracing::info!( + job_id = %job.id, + mode = %mode, + num_steps = num_steps, + steps = ?steps.iter().map(|s| &s.name).collect::>(), + "WAC v2 dispatching child jobs" + ); + + // Create child jobs for each step. + // Each child re-runs the full workflow with a checkpoint containing + // _executing_key = step_key, so only that step runs its inner function. + // + // IMPORTANT: To prevent a race condition where a fast child completes + // before the parent is suspended, we: + // 1. Pre-generate child UUIDs + // 2. Save checkpoint + suspend parent + seed child checkpoints + // 3. THEN push the child jobs (making them visible to workers) + + // Read the parent's original args for the child jobs + let parent_args: HashMap> = { + let stored: serde_json::Map = checkpoint.input_args.clone(); + if stored.is_empty() { + // First dispatch — read from the parent job's args + let row: Option = sqlx::query_scalar( + "SELECT args FROM v2_job WHERE id = $1 AND workspace_id = $2", + ) + .bind(&job.id) + .bind(&job.workspace_id) + .fetch_optional(db) + .await?; + let args_val = row.unwrap_or(Value::Object(Default::default())); + if let Value::Object(map) = args_val { + // Store for future re-runs + checkpoint.input_args = map.clone(); + map.into_iter() + .map(|(k, v)| { + let raw = serde_json::value::to_raw_value(&v).map_err(|e| { + error::Error::internal_err(format!( + "Failed to serialize arg '{k}': {e}" + )) + })?; + Ok((k, raw)) + }) + .collect::>>()? + } else { + HashMap::new() + } + } else { + stored + .into_iter() + .map(|(k, v)| { + let raw = serde_json::value::to_raw_value(&v).map_err(|e| { + error::Error::internal_err(format!( + "Failed to serialize arg '{k}': {e}" + )) + })?; + Ok((k, raw)) + }) + .collect::>>()? + } + }; + + // Pre-generate child UUIDs so we can save them in the checkpoint + // before the children become visible to workers. + // Validate key uniqueness — duplicate keys would cause one child's + // UUID to be overwritten in the job_ids map, making it unmappable + // on completion (the parent would hang). + { + let mut seen_keys = std::collections::HashSet::new(); + for s in &steps { + if !seen_keys.insert(&s.key) { + return Err(error::Error::internal_err(format!( + "WAC v2 duplicate step key '{}' — each task call must produce a unique key", + s.key + ))); + } + } + } + let job_ids: Vec<(String, Uuid)> = steps + .iter() + .map(|s| (s.key.clone(), ulid::Ulid::new().into())) + .collect(); + + // Resolve job_payload once (same for all children since they re-run + // the parent script) + let job_payload_template = match job.kind { + JobKind::Script => { + if let Some(hash) = job.runnable_id { + Ok(JobPayload::ScriptHash { + hash, + path: job.runnable_path.clone().unwrap_or_default(), + cache_ttl: job.cache_ttl, + cache_ignore_s3_path: job.cache_ignore_s3_path, + dedicated_worker: None, + language: job.script_lang.unwrap_or(ScriptLang::Bun), + priority: job.priority, + apply_preprocessor: false, + concurrency_settings: ConcurrencySettings::default(), + debouncing_settings: DebouncingSettings::default(), + }) + } else { + Err(error::Error::internal_err( + "WAC v2 Script job missing runnable_id".to_string(), + )) + } + } + JobKind::Preview => { + let row: Option<(Option, Option)> = sqlx::query_as( + "SELECT raw_code, raw_lock FROM v2_job WHERE id = $1 AND workspace_id = $2", + ) + .bind(&job.id) + .bind(&job.workspace_id) + .fetch_optional(db) + .await?; + let (code, lock) = row.unwrap_or_default(); + Ok(JobPayload::Code(RawCode { + content: code.unwrap_or_default(), + path: job.runnable_path.clone(), + hash: None, + language: job.script_lang.unwrap_or(ScriptLang::Bun), + lock: lock, + cache_ttl: job.cache_ttl, + cache_ignore_s3_path: job.cache_ignore_s3_path, + dedicated_worker: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), + debouncing_settings: DebouncingSettings::default(), + })) + } + _ => Err(error::Error::internal_err(format!( + "WAC v2 unsupported job kind: {:?}", + job.kind + ))), + }?; + + // Step 1: Save checkpoint, suspend parent, and seed child checkpoints + // in a single transaction — all BEFORE children become visible. + { + let mut tx = db.begin().await?; + + // Update checkpoint with pending steps + update_checkpoint_for_dispatch(&mut checkpoint, &steps, &mode, &job_ids); + let status_json = serde_json::to_value(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(&job.id) + .bind(&status_json) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!("Failed to save WAC checkpoint: {e}")) + })?; + + // Store per-child-job info for the WorkflowTimeline UI + for (step, (_, child_id)) in steps.iter().zip(job_ids.iter()) { + let child_id_str = child_id.to_string(); + let timeline_val = serde_json::json!({ + "scheduled_for": chrono::Utc::now().to_rfc3339(), + "name": step.key, + }); + sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( + COALESCE(workflow_as_code_status, '{}'::jsonb), + ARRAY[$2], + $3 + ) WHERE id = $1", + ) + .bind(&job.id) + .bind(&child_id_str) + .bind(&timeline_val) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Failed to update WAC timeline status: {e}" + )) + })?; + } + + // Suspend parent before children become visible. + // Keep running = true so the normal pull query ignores it. + // The suspended pull query picks it up when suspend reaches 0 + // (it checks: suspend_until IS NOT NULL AND suspend <= 0). + let suspend_count = num_steps as i32; + sqlx::query!( + "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 day' WHERE id = $1", + job.id, + suspend_count, + ) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Failed to suspend WAC parent job {}: {e}", + job.id + )) + })?; + + tx.commit().await?; + } + + // Step 2: Push child jobs (now visible to workers). + // Parent is already suspended, so child completions are safe. + // Track successfully pushed children so we can cancel them on + // partial failure (e.g. pushing child 3 of 5 fails). + let mut pushed_ids: Vec = Vec::with_capacity(num_steps); + let push_result: error::Result<()> = async { + for (step, (_, child_uuid)) in steps.iter().zip(job_ids.iter()) { + // Resolve job payload based on dispatch_type + let (job_payload, child_args, is_external) = match step.dispatch_type.as_str() { + "script" => { + // Resolve script path to job payload (handles hash, lang, etc.) + let (payload, _, _, _, _) = script_path_to_payload( + &step.script, + None, // no authed db for background workers + db.clone(), + &job.workspace_id, + Some(true), // skip preprocessor + ) + .await?; + let step_args: HashMap> = step + .args + .iter() + .map(|(k, v)| { + let raw = serde_json::value::to_raw_value(v).unwrap(); + (k.clone(), raw) + }) + .collect(); + (payload, step_args, true) + } + "flow" => { + let flow_info = get_latest_flow_version_info_for_path( + None, + db, + &job.workspace_id, + &step.script, + true, + ) + .await?; + let payload = JobPayload::Flow { + path: step.script.clone(), + dedicated_worker: flow_info.dedicated_worker, + apply_preprocessor: false, + version: flow_info.version, + }; + let step_args: HashMap> = step + .args + .iter() + .map(|(k, v)| { + let raw = serde_json::value::to_raw_value(v).unwrap(); + (k.clone(), raw) + }) + .collect(); + (payload, step_args, true) + } + _ => { + // "inline" — re-run parent with _executing_key + (job_payload_template.clone(), parent_args.clone(), false) + } + }; + + let push_args = PushArgs { args: &child_args, extra: None }; + + // Apply step-level overrides to payload (cache, concurrency) + let mut job_payload = job_payload; + if let Some(cache_ttl) = step.cache_ttl { + match &mut job_payload { + JobPayload::ScriptHash { cache_ttl: ref mut ct, .. } => { + *ct = Some(cache_ttl) + } + JobPayload::Code(ref mut code) => code.cache_ttl = Some(cache_ttl), + _ => {} + } + } + if step.concurrent_limit.is_some() + || step.concurrency_key.is_some() + || step.concurrency_time_window_s.is_some() + { + match &mut job_payload { + JobPayload::ScriptHash { concurrency_settings: ref mut cs, .. } => { + if let Some(limit) = step.concurrent_limit { + cs.concurrent_limit = Some(limit); + } + if let Some(ref key) = step.concurrency_key { + cs.concurrency_key = Some(key.clone()); + } + if let Some(window) = step.concurrency_time_window_s { + cs.concurrency_time_window_s = Some(window); + } + } + JobPayload::Code(ref mut code) => { + if let Some(limit) = step.concurrent_limit { + code.concurrency_settings.concurrent_limit = Some(limit); + } + if let Some(ref key) = step.concurrency_key { + code.concurrency_settings.custom_concurrency_key = + Some(key.clone()); + } + if let Some(window) = step.concurrency_time_window_s { + code.concurrency_settings.concurrency_time_window_s = + Some(window); + } + } + _ => {} + } + } + + let (_, mut tx) = push( + db, + PushIsolationLevel::IsolatedRoot(db.clone()), + &job.workspace_id, + job_payload, + push_args, + &job.created_by, + &job.permissioned_as_email, + job.permissioned_as.clone(), + None, + None, + None, + Some(job.id), // parent_job + job.root_job.or(Some(job.id)), // root_job + job.flow_innermost_root_job, + Some(*child_uuid), // pre-generated job_id + false, // is_flow_step + false, // same_worker + None, // pre_run_error + job.visible_to_owner, + step.tag.clone().or_else(|| Some(job.tag.clone())), + step.timeout.or(job.timeout), + None, // flow_step_id + step.priority, // priority_override + None, // authed + false, // running + None, // end_user_email + None, // trigger + None, // suspended_mode + ) + .await?; + + // Seed child checkpoint only for inline tasks (they need + // _executing_key to know which step to run). External + // scripts/flows don't need a WAC checkpoint. + if !is_external { + let child_checkpoint_json = serde_json::json!({ + "completed_steps": &checkpoint.completed_steps, + "_executing_key": &step.key, + }); + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(child_uuid) + .bind(&child_checkpoint_json) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Failed to seed child checkpoint: {e}" + )) + })?; + } + + tx.commit().await.map_err(|e| { + error::Error::internal_err(format!("Failed to commit child push: {e}")) + })?; + + pushed_ids.push(*child_uuid); + + tracing::info!( + parent_job = %job.id, + child_job = %child_uuid, + step_name = %step.name, + step_key = %step.key, + "WAC v2 dispatched child job" + ); + } + Ok(()) + } + .await; + + if let Err(e) = push_result { + tracing::error!( + job_id = %job.id, + error = %e, + pushed_count = pushed_ids.len(), + total_count = num_steps, + "WAC v2 failed to push child jobs, cleaning up" + ); + + // Cancel already-pushed children so they don't complete and + // corrupt the checkpoint (they'd decrement suspend on a parent + // that's about to be unsuspended and re-run). + for child_id in &pushed_ids { + let _ = sqlx::query!( + "UPDATE v2_job_queue SET canceled_by = $2, canceled_reason = $3 WHERE id = $1", + child_id, + "system", + "WAC dispatch failed: not all children could be pushed", + ) + .execute(db) + .await; + } + + // Clear pending_steps from checkpoint so the parent doesn't + // think children are outstanding when it re-runs. + let _ = sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = \ + workflow_as_code_status #- '{_checkpoint,pending_steps}' \ + WHERE id = $1", + ) + .bind(&job.id) + .execute(db) + .await; + + // Unsuspend parent so the error propagates instead of a 14-day hang + let _ = sqlx::query!( + "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", + job.id, + ) + .execute(db) + .await; + return Err(e); + } + + tracing::info!( + job_id = %job.id, + num_steps = num_steps, + "WAC v2 parent job suspended" + ); + + Err(error::Error::WacSuspended(format!( + "WAC v2 job {} suspended waiting for {} child job(s)", + job.id, num_steps + ))) + } + WacOutput::Approval { key, timeout, form } => { + let db = match conn { + Connection::Sql(db) => db, + _ => { + return Err(error::Error::internal_err( + "WAC v2 approval requires SQL connection".to_string(), + )) + } + }; + + let mut checkpoint = load_checkpoint(db, &job.id).await?; + let timeout_secs = timeout.unwrap_or(1800) as f64; + + // Mark this step as pending approval + checkpoint.pending_steps = Some(crate::wac_executor::WacPendingSteps { + mode: "approval".to_string(), + keys: vec![key.clone()], + job_ids: serde_json::Map::new(), + }); + + let mut tx = db.begin().await?; + + // Save checkpoint + let status_json = serde_json::to_value(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(&job.id) + .bind(&status_json) + .execute(&mut *tx) + .await + .map_err(|e| error::Error::internal_err(format!("Failed to save checkpoint: {e}")))?; + + // Store approval form metadata for the approval page endpoint + let approval_meta = serde_json::json!({ + "key": key, + "form": form, + "timeout": timeout_secs as u32, + }); + sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( + COALESCE(workflow_as_code_status, '{}'::jsonb), + '{_approval}', + $2::jsonb + ) WHERE id = $1", + ) + .bind(&job.id) + .bind(&approval_meta) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!("Failed to save approval meta: {e}")) + })?; + + // Suspend parent with suspend=1 (waiting for 1 approval event) + sqlx::query!( + "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1", + job.id, + timeout_secs, + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + tracing::info!( + job_id = %job.id, + approval_key = %key, + timeout_secs = timeout_secs, + "WAC v2 parent job suspended waiting for approval" + ); + + Err(error::Error::WacSuspended(format!( + "WAC v2 job {} suspended waiting for approval (key: {})", + job.id, key + ))) + } + WacOutput::Sleep { key, seconds } => { + let db = match conn { + Connection::Sql(db) => db, + _ => { + return Err(error::Error::internal_err( + "WAC v2 sleep requires SQL connection".to_string(), + )) + } + }; + + let mut checkpoint = load_checkpoint(db, &job.id).await?; + let sleep_secs = seconds.max(1) as f64; + + // Mark this step as pending sleep + checkpoint.pending_steps = Some(crate::wac_executor::WacPendingSteps { + mode: "sleep".to_string(), + keys: vec![key.clone()], + job_ids: serde_json::Map::new(), + }); + + let mut tx = db.begin().await?; + + // Save checkpoint + let status_json = serde_json::to_value(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(&job.id) + .bind(&status_json) + .execute(&mut *tx) + .await + .map_err(|e| error::Error::internal_err(format!("Failed to save checkpoint: {e}")))?; + + // Suspend parent — it will auto-resume when suspend_until passes. + // Use suspend=1 (not 0) so the suspended pull query only picks it up + // when `suspend_until <= now()`, not via `suspend <= 0`. + sqlx::query!( + "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1", + job.id, + sleep_secs, + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + tracing::info!( + job_id = %job.id, + sleep_key = %key, + sleep_secs = sleep_secs, + "WAC v2 parent job sleeping for {}s", + sleep_secs + ); + + Err(error::Error::WacSuspended(format!( + "WAC v2 job {} sleeping for {}s (key: {})", + job.id, seconds, key + ))) + } + WacOutput::InlineCheckpoint { key, result: value } => { + let db = match conn { + Connection::Sql(db) => db, + _ => { + return Err(error::Error::internal_err( + "WAC v2 inline checkpoint requires SQL connection".to_string(), + )) + } + }; + + let mut checkpoint = load_checkpoint(db, &job.id).await?; + + // Source hash validation (same as Dispatch path) + let current_hash = job.runnable_id.map(|h| h.0.to_string()).unwrap_or_default(); + if !current_hash.is_empty() { + if checkpoint.source_hash.is_empty() { + checkpoint.source_hash = current_hash.clone(); + } else if checkpoint.source_hash != current_hash { + return Err(error::Error::ExecutionErr( + "Workflow source code changed between replays. \ + Cannot safely resume from checkpoint — step keys may have shifted. \ + Please restart this workflow." + .to_string(), + )); + } + } + + tracing::info!( + job_id = %job.id, + step_key = %key, + "WAC v2 inline checkpoint — persisting step result" + ); + + add_completed_step(&mut checkpoint, &key, value); + + // Save checkpoint + reset running in a single transaction + { + let mut tx = db.begin().await?; + let status_json = serde_json::to_value(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(&job.id) + .bind(&status_json) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!("Failed to save WAC checkpoint: {e}")) + })?; + + // Reset running=false so the job is immediately eligible for pickup. + // Unlike dispatch (which sets suspend>0), inline checkpoints don't suspend — + // the job should be re-run right away to continue past the cached step. + sqlx::query!( + "UPDATE v2_job_queue SET running = false, started_at = null WHERE id = $1", + job.id, + ) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Failed to reset running state for inline checkpoint: {e}" + )) + })?; + + tx.commit().await?; + } + + Err(error::Error::WacSuspended(format!( + "WAC v2 job {} inline checkpoint for step {}", + job.id, key + ))) + } + } } pub async fn get_common_bun_proc_envs(base_internal_url: Option<&str>) -> HashMap { diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 1239aa8d7d..846c421024 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -886,7 +886,7 @@ pub async fn cached_result_path( } #[cfg(feature = "parquet")] -async fn get_workspace_s3_resource_path( +pub(crate) async fn get_workspace_s3_resource_path( db: &DB, client: &AuthedClient, workspace_id: &str, @@ -948,7 +948,11 @@ async fn get_workspace_s3_resource_path( ) } Some(LargeFileStorage::FilesystemStorage(fs)) => { - (StorageResourceType::Filesystem, fs.root_path.clone()) + return Ok(Some( + windmill_object_store::ObjectStoreResource::Filesystem( + windmill_object_store::FilesystemSettings { root_path: fs.root_path.clone() }, + ), + )); } None => { return Ok(None); diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index 4c3e63f4c9..22f46e724b 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -600,7 +600,10 @@ pub async fn handle_csharp_job( .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::CSharp).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -633,7 +636,10 @@ pub async fn handle_csharp_job( .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::CSharp).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 19831eadd0..d9f6785eb6 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -17,6 +17,7 @@ use crate::{ NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, }; use windmill_common::client::AuthedClient; +use windmill_common::worker::TypeScriptAnnotations; use tokio::{fs::File, io::AsyncReadExt, process::Command}; use windmill_common::{error::Result, scripts::ScriptLang, worker::write_file, BASE_URL}; @@ -120,11 +121,13 @@ async fn get_common_deno_proc_envs( } // Add proxy envs (including OTEL tracing proxy if enabled for deno) - for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno) - .await - .unwrap_or_default() - { - deno_envs.insert(k.to_string(), v); + if let Some(conn) = conn { + for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno, job_id, w_id, conn) + .await + .unwrap_or_default() + { + deno_envs.insert(k.to_string(), v); + } } return deno_envs; @@ -231,8 +234,13 @@ pub async fn handle_deno_job( occupancy_metrics: &mut OccupancyMetrics, has_stream: &mut bool, ) -> error::Result> { + let annotations = TypeScriptAnnotations::parse(inner_content); + // let mut start = Instant::now(); - let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); + let mut logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); + if annotations.sandbox { + logs1.push_str("sandbox mode (nsjail)\n"); + } append_logs(&job.id, &job.workspace_id, logs1, conn).await; let main_override = job.script_entrypoint_override.as_deref(); @@ -451,7 +459,7 @@ try {{ for flag in deno_flags { args.push(flag); } - } else if is_sandboxing_enabled() { + } else if is_sandboxing_enabled() || annotations.sandbox { args.push("--allow-net"); args.push("--allow-sys"); args.push(allow_read.as_str()); diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 29fff1a2ba..0315bf25c3 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -53,8 +53,7 @@ fn get_windows_program_files() -> String { #[cfg(windows)] fn windows_gopath() -> String { - let tmp_dir = get_windows_tmp_dir(); - GO_CACHE_DIR.replace("/tmp", &tmp_dir).replace("/", r"\\") + GO_CACHE_DIR.replace('/', "\\") } #[cfg(windows)] @@ -355,7 +354,7 @@ func Run(req Req) (interface{{}}, error){{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Go).await?) + .envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -376,7 +375,7 @@ func Run(req Req) (interface{{}}, error){{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Go).await?) + .envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 1a107990dc..205da6700a 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -737,21 +737,24 @@ where let update_job_row = i == 2 || (!*SLOW_LOGS && (i < 20 || (i < 120 && i % 5 == 0) || i % 10 == 0)) || i % 20 == 0; if update_job_row && job_id != Uuid::nil() { if let Connection::Sql(ref db) = conn { - // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs - if i == 2 { - memory_metric_id = job_metrics::register_metric_for_job( - &db, - w_id.to_string(), - job_id, - "memory_kb".to_string(), - job_metrics::MetricKind::TimeseriesInt, - Some("Job Memory Footprint (kB)".to_string()), - ) - .await; - } - if let Ok(ref metric_id) = memory_metric_id { - if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await { - tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); + // Only track memory when it's non-zero (avoids storing all-zero timeseries for jobs that don't report memory) + if current_mem > 0 { + // Register on first non-zero reading (deferred from i==2 to avoid metric for jobs with no memory reporting) + if memory_metric_id.is_err() { + memory_metric_id = job_metrics::register_metric_for_job( + &db, + w_id.to_string(), + job_id, + "memory_kb".to_string(), + job_metrics::MetricKind::TimeseriesInt, + Some("Job Memory Footprint (kB)".to_string()), + ) + .await; + } + if let Ok(ref metric_id) = memory_metric_id { + if let Err(err) = job_metrics::record_timeseries_value(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem), job_metrics::MetricKind::TimeseriesInt).await { + tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); + } } } } diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index efbe403145..6f558e3328 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -618,6 +618,7 @@ async fn run<'a>( .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)) .args(vec![ "--config", "run.config.proto", @@ -675,7 +676,8 @@ async fn run<'a>( .env("HOME", &*JAVA_HOME_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) - .envs(reserved_variables); + .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)); if metadata(TRUST_STORE_PATH.clone()).await.is_ok() { cmd.args(&[ &format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH), diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 741458c0e0..2fe56f50b9 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -70,6 +70,10 @@ mod sanitized_sql_params; mod schema; pub mod sql_utils; mod universal_pkg_installer; +#[cfg(feature = "private")] +mod volume_ee; +mod volume_oss; +pub mod wac_executor; mod worker; mod worker_flow; mod worker_lockfiles; diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index ca309105b5..28ac27c925 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -264,7 +264,7 @@ async fn run<'a>( .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Nu).await?) + .envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?) .args(vec![ "--config", "run.config.proto", @@ -303,7 +303,7 @@ async fn run<'a>( .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Nu).await?) + .envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?) // TODO(v1): // "--plugins", // &format!( diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index e98755e555..df006fef7f 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -22,7 +22,7 @@ use crate::{ get_reserved_variables, read_result, start_child_process, MaybeLock, OccupancyMetrics, }, handle_child::handle_child, - COMPOSER_CACHE_DIR, COMPOSER_PATH, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, + is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, }; use windmill_common::client::AuthedClient; @@ -316,6 +316,7 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)) .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .args(args) @@ -332,6 +333,7 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)) .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .stdin(Stdio::null()) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index b289f05380..c74feb0a0a 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -134,8 +134,8 @@ use crate::{ handle_child::handle_child, is_sandboxing_enabled, read_ee_registry, worker_utils::ping_job_status, - PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, - PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR, + PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, + PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR, UV_INDEX_STRATEGY, }; use windmill_common::client::AuthedClient; @@ -567,6 +567,17 @@ pub async fn handle_python_job( let annotations = PythonAnnotations::parse(inner_content); + let is_wac_v2 = job.script_entrypoint_override.is_none() + && crate::wac_executor::is_wac_v2_py(inner_content); + + if annotations.sandbox && NSJAIL_AVAILABLE.is_none() { + return Err(Error::ExecutionErr( + "Script has #sandbox annotation but nsjail is not available on this worker. \ + Please ensure nsjail is installed or remove the #sandbox annotation." + .to_string(), + )); + } + let (py_version, mut additional_python_paths) = handle_python_deps( job_dir, requirements_o, @@ -605,16 +616,14 @@ pub async fn handle_python_job( } { - append_logs( - &job.id, - &job.workspace_id, - format!( - "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", - py_version.clone().to_string() - ), - conn, - ) - .await; + let mut logs = format!( + "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", + py_version.clone().to_string() + ); + if annotations.sandbox { + logs.push_str("sandbox mode (nsjail)\n"); + } + append_logs(&job.id, &job.workspace_id, logs, conn).await; } let ( import_loader, @@ -671,8 +680,68 @@ pub async fn handle_python_job( String::new() }; let main_override = main_name.unwrap_or_else(|| "main".to_string()); - let wrapper_content: String = format!( - r#" + let wrapper_content: String = if is_wac_v2 { + format!( + r#" +import os +import json +{import_loader} +{import_base64} +{import_datetime} +import traceback +import sys +from {module_dir_dot} import {last} as inner_script +from wmill.client import _run_workflow + +with open("args.json") as f: + kwargs = json.load(f, strict=False) +args = {{}} +{transforms} + +with open("checkpoint.json") as f: + checkpoint = json.load(f, strict=False) + +result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.json") + +# Find the @workflow-decorated function +workflow_fn = None +for name in dir(inner_script): + obj = getattr(inner_script, name) + if callable(obj) and getattr(obj, '_is_workflow', False): + workflow_fn = obj + break + +if workflow_fn is None: + raise ValueError("No @workflow function found in script") + +for k, v in list(args.items()): + if v == '': + del args[k] + +try: + output = _run_workflow(workflow_fn, checkpoint, args) + output_json = json.dumps(output, separators=(',', ':'), default=str) + with open(result_json, 'w') as f: + f.write(output_json) +except BaseException as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + tb = traceback.format_tb(exc_traceback) + with open(result_json, 'w') as f: + err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }} + extra = e.__dict__ + if extra and len(extra) > 0: + err['extra'] = extra + flow_node_id = os.environ.get('WM_FLOW_STEP_ID') + if flow_node_id: + err['step_id'] = flow_node_id + err_json = json.dumps(err, separators=(',', ':'), default=str).replace('\n', '') + f.write(err_json) + sys.exit(1) +"#, + ) + } else { + format!( + r#" import os import json {import_loader} @@ -745,9 +814,26 @@ except BaseException as e: f.write(err_json) sys.exit(1) "#, - ); + ) + }; write_file(job_dir, "wrapper.py", &wrapper_content)?; + // For WAC v2, write checkpoint.json before python runs. + if is_wac_v2 { + if let Connection::Sql(db) = conn { + let checkpoint = crate::wac_executor::load_checkpoint(db, &job.id).await?; + let checkpoint = + crate::wac_executor::prepare_checkpoint_for_resume(db, &job.id, checkpoint).await?; + + let checkpoint_json = serde_json::to_string(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + write_file(job_dir, "checkpoint.json", &checkpoint_json)?; + } else { + write_file(job_dir, "checkpoint.json", r#"{"completed_steps":{}}"#)?; + } + } + tracing::debug!("Finished writing wrapper"); let mut reserved_variables = @@ -784,7 +870,7 @@ except BaseException as e: #[cfg(windows)] let additional_python_paths_folders = additional_python_paths_folders.replace(":", ";"); - if is_sandboxing_enabled() { + if is_sandboxing_enabled() || annotations.sandbox { let shared_deps = additional_python_paths .into_iter() .map(|pp| { @@ -828,14 +914,17 @@ mount {{ job.id ); - let child = if is_sandboxing_enabled() { + let child = if is_sandboxing_enabled() || annotations.sandbox { let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd .current_dir(job_dir) .env_clear() // inject PYTHONPATH here - for some reason I had to do it in nsjail conf .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Python3).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -861,7 +950,10 @@ mount {{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Python3).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -924,7 +1016,15 @@ mount {{ *new_args = Some(args.clone()); } - read_result(job_dir, handle_result.result_stream).await + let result = read_result(job_dir, handle_result.result_stream).await?; + + // WAC v2 post-execution: parse output and handle dispatch/suspend. + // Box::pin to avoid bloating handle_python_job's async state machine (stack overflow). + if is_wac_v2 { + return Box::pin(crate::bun_executor::handle_wac_v2_output(result, job, conn)).await; + } + + Ok(result) } async fn prepare_wrapper( diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index c237d8cc02..e26633fc9a 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -665,7 +665,7 @@ pub async fn process_completed_job( add_time!(bench, "pre add_completed_job"); - let (_, duration) = add_completed_job( + let (_, duration, wac_job_ids) = add_completed_job( db, &job, true, @@ -717,6 +717,29 @@ pub async fn process_completed_job( } return Ok(r); } + } else if let Some(parent_job) = parent_job { + // wac_job_ids is piggybacked from the duration write in + // add_completed_job — no extra query needed. + if let Some(job_ids) = wac_job_ids { + if let Ok(Some(_)) = handle_wac_child_completion( + db, + &job_id, + parent_job, + &workspace_id, + result, + true, + job_ids, + ) + .await + { + if let Some(done_tx) = done_tx { + done_tx + .send(()) + .expect("done receiver should still be alive"); + } + return Ok(None); + } + } } } else { let result = add_completed_job_error( @@ -770,11 +793,229 @@ pub async fn process_completed_job( } return Ok(r); } + } else if let Some(parent_job) = job.parent_job { + // WAC child failed — query job_ids from parent (errors are rare, + // so the extra read is acceptable here). + let job_ids_json: Option> = sqlx::query_scalar( + "SELECT workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' \ + FROM v2_job_status WHERE id = $1", + ) + .bind(&parent_job) + .fetch_optional(db) + .await?; + if let Some(Some(job_ids)) = job_ids_json { + let err_result = Arc::new(serde_json::value::to_raw_value(&result).unwrap()); + if let Ok(Some(_)) = handle_wac_child_completion( + db, + &job.id, + parent_job, + &job.workspace_id, + err_result, + false, + job_ids, + ) + .await + { + if let Some(done_tx) = done_tx { + done_tx + .send(()) + .expect("done receiver should still be alive"); + } + return Ok(None); + } + } } } return Ok(None); } +/// Handle a WAC v2 child job completion. +/// Returns Ok(Some(())) if the parent was a WAC job and was handled, +/// Ok(None) if the parent is not a WAC job (caller should fall through). +/// +/// CONCURRENCY: Multiple parallel children may complete simultaneously on +/// different workers. We use atomic SQL operations throughout: +/// - `completed_steps` is merged via `jsonb_set(... || jsonb_build_object(...))` +/// — PostgreSQL serialises concurrent UPDATEs on the same row, so each +/// worker sees the previous worker's writes. +/// - The suspend counter (set to N at dispatch time) is decremented atomically +/// with `RETURNING` to determine the "all done" condition. +pub(crate) async fn handle_wac_child_completion( + db: &DB, + child_job_id: &Uuid, + parent_job_id: Uuid, + workspace_id: &str, + result: Arc>, + success: bool, + job_ids_value: Value, +) -> error::Result> { + let job_ids = match job_ids_value { + Value::Object(m) => m, + _ => return Ok(None), // Not a WAC parent or no pending steps + }; + + let child_id_str = child_job_id.to_string(); + let step_key = job_ids.iter().find_map(|(key, val)| { + if val.as_str() == Some(&child_id_str) { + Some(key.clone()) + } else { + None + } + }); + + let step_key = match step_key { + Some(k) => k, + None => { + if !success { + // No step key and failed — can't store error, fail parent immediately + tracing::error!( + parent_job = %parent_job_id, + child_job = %child_job_id, + "WAC v2 child job failed but no step key found, failing parent" + ); + sqlx::query!( + "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", + parent_job_id, + ) + .execute(db) + .await?; + let parent_mini = get_mini_completed_job(&parent_job_id, workspace_id, db).await?; + if let Some(parent_mini) = parent_mini { + let child_err: Value = + serde_json::from_str(result.get()).unwrap_or(Value::Null); + let err_value = json!({ + "message": format!("WAC child job {} failed (no step key)", child_job_id), + "error": child_err, + }); + let _ = windmill_queue::add_completed_job_error( + db, + &parent_mini, + 0, + None, + err_value, + "wac_child_handler", + false, + None, + ) + .await; + } + return Ok(Some(())); + } + tracing::warn!( + parent_job = %parent_job_id, + child_job = %child_job_id, + "WAC v2 child completed but no matching step key found in checkpoint, decrementing suspend to avoid parent hang" + ); + // Still decrement suspend so the parent doesn't hang indefinitely + let _ = sqlx::query_scalar!( + "UPDATE v2_job_queue \ + SET suspend = GREATEST(suspend - 1, 0) \ + WHERE id = $1 \ + RETURNING suspend", + parent_job_id, + ) + .fetch_optional(db) + .await?; + return Ok(Some(())); + } + }; + + // Build result — wrap errors with _error marker so workflow try/catch can handle them + let result_value: Value = if success { + serde_json::from_str(result.get()).unwrap_or(Value::Null) + } else { + let child_err: Value = serde_json::from_str(result.get()).unwrap_or(Value::Null); + tracing::info!( + parent_job = %parent_job_id, + child_job = %child_job_id, + step_key = %step_key, + "WAC v2 child job failed, storing error for workflow try/catch" + ); + json!({ + "__wmill_error": true, + "message": format!("WAC task '{}' failed (child job {})", step_key, child_job_id), + "child_job_id": child_job_id.to_string(), + "step_key": step_key, + "result": child_err, + }) + }; + + tracing::info!( + parent_job = %parent_job_id, + child_job = %child_job_id, + step_key = %step_key, + success = success, + "WAC v2 child job completed" + ); + + // Use a transaction to ensure completed_steps merge + suspend decrement + // are atomic. Without this, a crash between the two could strand the parent. + let result_json = serde_json::to_value(&result_value) + .map_err(|e| error::Error::InternalErr(format!("Failed to serialize step result: {e}")))?; + + let mut tx = db.begin().await?; + + // Merge the completed step into the checkpoint. + // Uses `|| jsonb_build_object(key, value)` so concurrent children on + // different workers don't overwrite each other — PostgreSQL serialises + // concurrent UPDATEs on the same row and each sees the previous write. + sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( + workflow_as_code_status, + '{_checkpoint,completed_steps}', + COALESCE(workflow_as_code_status->'_checkpoint'->'completed_steps', '{}'::jsonb) + || jsonb_build_object($2::text, $3::jsonb) + ) WHERE id = $1", + ) + .bind(&parent_job_id) + .bind(&step_key) + .bind(&result_json) + .execute(&mut *tx) + .await + .map_err(|e| error::Error::InternalErr(format!("Failed to add WAC completed step: {e}")))?; + + // Decrement the suspend counter. The counter was set to N (number of + // children) at dispatch time. When it reaches 0 all children are done. + // Keep suspend_until non-null so the suspended pull query + // (`WHERE suspend_until IS NOT NULL AND suspend <= 0`) picks up the parent. + let new_suspend: Option = sqlx::query_scalar!( + "UPDATE v2_job_queue \ + SET suspend = GREATEST(suspend - 1, 0) \ + WHERE id = $1 \ + RETURNING suspend", + parent_job_id, + ) + .fetch_optional(&mut *tx) + .await?; + + let all_done = new_suspend == Some(0); + + if all_done { + // Clear pending_steps from checkpoint since all children are complete. + // This is cosmetic — the next replay will overwrite it anyway — but + // keeps the checkpoint clean for frontend display. + let _ = sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = \ + workflow_as_code_status #- '{_checkpoint,pending_steps}' \ + WHERE id = $1", + ) + .bind(&parent_job_id) + .execute(&mut *tx) + .await; + } + + tx.commit().await?; + + if all_done { + tracing::info!( + parent_job = %parent_job_id, + "WAC v2 all child jobs completed, unsuspending parent" + ); + } + + Ok(Some(())) +} + pub async fn handle_non_flow_job_error( db: &DB, job: &MiniCompletedJob, diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index 11031fa481..d283b2a1c4 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -812,7 +812,10 @@ mount {{ .envs(envs) .envs(reserved_variables) .envs(RUBY_PROXY_ENVS.clone()) - .envs(get_proxy_envs_for_lang(&ScriptLang::Ruby).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn) + .await?, + ) .args(vec![ "--config", "run.config.proto", @@ -851,7 +854,10 @@ mount {{ .env("BASE_INTERNAL_URL", base_internal_url) .envs(reserved_variables) .envs(RUBY_PROXY_ENVS.clone()) - .envs(get_proxy_envs_for_lang(&ScriptLang::Ruby).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn) + .await?, + ) .envs(envs); cmd.stdin(Stdio::null()) diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 216c59f2cd..dae8e08766 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -41,20 +41,30 @@ const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.co const NSJAIL_CONFIG_COMPILE_RUST_CONTENT: &str = include_str!("../nsjail/download.rust.config.proto"); +#[cfg(windows)] +const RUST_BIN_NAME: &str = "main.exe"; +#[cfg(not(windows))] +const RUST_BIN_NAME: &str = "main"; + fn find_cargo_path() -> String { if let Ok(p) = std::env::var("CARGO_PATH") { return p; } - let from_home = format!("{}/bin/cargo", CARGO_HOME.as_str()); - if std::path::Path::new(&from_home).exists() { - return from_home; - } - for p in ["/usr/local/cargo/bin/cargo", "/usr/bin/cargo"] { + let candidates = if cfg!(windows) { + vec![format!("{}\\bin\\cargo.exe", CARGO_HOME.as_str())] + } else { + vec![ + format!("{}/bin/cargo", CARGO_HOME.as_str()), + "/usr/local/cargo/bin/cargo".to_string(), + "/usr/bin/cargo".to_string(), + ] + }; + for p in &candidates { if std::path::Path::new(p).exists() { - return p.to_string(); + return p.clone(); } } - from_home + candidates.into_iter().next().unwrap() } #[cfg(not(windows))] @@ -71,7 +81,6 @@ fn find_preinstalled_dir(env_var: &str, candidates: &[&str]) -> String { } lazy_static::lazy_static! { - static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable"); static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_HOME_DEFAULT.clone() }); static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| { RUSTUP_HOME_DEFAULT.clone() }); static ref CARGO_PATH: String = find_cargo_path(); @@ -81,14 +90,14 @@ lazy_static::lazy_static! { #[cfg(windows)] lazy_static::lazy_static! { - static ref CARGO_HOME_DEFAULT: String = format!("{}\\.cargo", *HOME_DIR); - static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", *HOME_DIR); + static ref CARGO_HOME_DEFAULT: String = format!("{}\\.cargo", HOME_ENV.as_str()); + static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", HOME_ENV.as_str()); } #[cfg(not(windows))] lazy_static::lazy_static! { - static ref CARGO_HOME_DEFAULT: String = format!("{}/.cargo", *HOME_DIR); - static ref RUSTUP_HOME_DEFAULT: String = format!("{}/.rustup", *HOME_DIR); + static ref CARGO_HOME_DEFAULT: String = format!("{}/.cargo", HOME_ENV.as_str()); + static ref RUSTUP_HOME_DEFAULT: String = format!("{}/.rustup", HOME_ENV.as_str()); } const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/"; @@ -97,11 +106,11 @@ const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/"; lazy_static::lazy_static! { static ref PREINSTALLED_CARGO: String = find_preinstalled_dir( "CARGO_PREINSTALL_DIR", - &["/usr/local/cargo", &format!("{}/.cargo", *HOME_DIR)], + &["/usr/local/cargo", &format!("{}/.cargo", HOME_ENV.as_str())], ); static ref PREINSTALLED_RUSTUP: String = find_preinstalled_dir( "RUSTUP_PREINSTALL_DIR", - &["/usr/local/rustup", &format!("{}/.rustup", *HOME_DIR)], + &["/usr/local/rustup", &format!("{}/.rustup", HOME_ENV.as_str())], ); } @@ -521,6 +530,13 @@ pub async fn build_rust_crate( std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), ); build_rust_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + // MSVC linker needs LIB and INCLUDE to find kernel32.lib etc. + if let Ok(lib) = std::env::var("LIB") { + build_rust_cmd.env("LIB", lib); + } + if let Ok(include) = std::env::var("INCLUDE") { + build_rust_cmd.env("INCLUDE", include); + } } start_child_process(build_rust_cmd, CARGO_PATH.as_str(), false).await? }; @@ -545,30 +561,29 @@ pub async fn build_rust_crate( tokio::fs::copy( &format!( - "{build_dir}/target/{}/main", + "{build_dir}/target/{}/{RUST_BIN_NAME}", if is_preview { "debug" } else { "release" }, ), - format! {"{job_dir}/main"}, + format!("{job_dir}/{RUST_BIN_NAME}"), ) .await .map_err(|e| { Error::ExecutionErr(format!( - "could not copy built binary from [...]/target/.../main to {job_dir}/main: {e:?}" + "could not copy built binary from [...]/target/.../{RUST_BIN_NAME} to {job_dir}/{RUST_BIN_NAME}: {e:?}" )) })?; match save_cache( &bin_path, &format!("{RUST_OBJECT_STORE_PREFIX}{hash}"), - &format!("{job_dir}/main"), + &format!("{job_dir}/{RUST_BIN_NAME}"), false, ) .await { Err(e) => { let em = format!( - "could not save {bin_path} to {} to rust cache: {e:?}", - format!("{job_dir}/main"), + "could not save {bin_path} to {job_dir}/{RUST_BIN_NAME} to rust cache: {e:?}", ); tracing::error!(em); Ok(em) @@ -618,16 +633,16 @@ pub async fn handle_rust_job( let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let cache_logs = if cache { - let target = format!("{job_dir}/main"); + let target = format!("{job_dir}/{RUST_BIN_NAME}"); #[cfg(unix)] let symlink = std::os::unix::fs::symlink(&bin_path, &target); #[cfg(windows)] - let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target); + let symlink = std::os::windows::fs::symlink_file(&bin_path, &target); symlink.map_err(|e| { Error::ExecutionErr(format!( - "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" + "could not copy cached binary from {bin_path} to {target}: {e:?}" )) })?; @@ -685,7 +700,10 @@ pub async fn handle_rust_job( .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Rust).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -694,14 +712,17 @@ pub async fn handle_rust_job( .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { - let compiled_executable_name = "./main"; + let compiled_executable_name = &format!("{job_dir}/{RUST_BIN_NAME}"); let mut run_rust = build_command_with_isolation(compiled_executable_name, &[]); run_rust .current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Rust).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 9f89f2fc44..90d06287bc 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -9,8 +9,8 @@ use serde_json::{json, value::RawValue, Value}; use sha2::{Digest, Sha256}; use std::collections::HashMap; use windmill_common::error::to_anyhow; -use windmill_object_store::convert_json_line_stream; use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; +use windmill_object_store::convert_json_line_stream; use windmill_common::{error::Error, worker::to_raw_value}; use windmill_parser_sql::{ diff --git a/backend/windmill-worker/src/volume_oss.rs b/backend/windmill-worker/src/volume_oss.rs new file mode 100644 index 0000000000..3736e83485 --- /dev/null +++ b/backend/windmill-worker/src/volume_oss.rs @@ -0,0 +1,112 @@ +#[cfg(feature = "private")] +pub(crate) use crate::volume_ee::*; + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) struct LeaseRenewalGuard(pub Option>); + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +impl Drop for LeaseRenewalGuard { + fn drop(&mut self) { + if let Some(handle) = self.0.take() { + handle.abort(); + } + } +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) struct VolumeSetupResult { + pub states: Vec, + pub writable: Vec, + pub client: Option>, + pub lease_renewal: LeaseRenewalGuard, +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +#[allow(dead_code)] +pub(crate) fn setup_volume_mount_paths( + _volume: &windmill_worker_volumes::VolumeMount, + _state: &windmill_worker_volumes::VolumeState, + _job_dir: &str, + _language: windmill_common::scripts::ScriptLang, + _envs: &mut std::collections::HashMap, + _shared_mount: &mut String, +) -> windmill_common::error::Result<()> { + Err(windmill_common::error::Error::internal_err( + "Volumes are not available in OSS".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn setup_volumes_sql_worker( + _volume_mounts: &[windmill_worker_volumes::VolumeMount], + _db: &windmill_common::DB, + _workspace_id: &str, + _job_id: uuid::Uuid, + _permissioned_as: &str, + _worker_name: &str, + _job_dir: &str, + _client: &windmill_common::client::AuthedClient, + _conn: &windmill_common::worker::Connection, + _language: windmill_common::scripts::ScriptLang, + _envs: &mut std::collections::HashMap, + _shared_mount: &mut String, +) -> windmill_common::error::Result { + Err(windmill_common::error::Error::internal_err( + "Volumes are not available in OSS".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn setup_volumes_http_worker( + _volume_mounts: &[windmill_worker_volumes::VolumeMount], + _http: &windmill_common::worker::HttpClient, + _workspace_id: &str, + _job_id: uuid::Uuid, + _permissioned_as: &str, + _canceled_by: &Option, + _worker_name: &str, + _job_dir: &str, + _conn: &windmill_common::worker::Connection, + _language: windmill_common::scripts::ScriptLang, + _envs: &mut std::collections::HashMap, + _shared_mount: &mut String, +) -> windmill_common::error::Result { + Err(windmill_common::error::Error::internal_err( + "Volumes are not available in OSS".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn sync_volumes_sql_worker( + _volume_states: &[windmill_worker_volumes::VolumeState], + _volume_writable: &[bool], + _vol_client: &std::sync::Arc, + _db: &windmill_common::DB, + _workspace_id: &str, + _job_id: uuid::Uuid, + _worker_name: &str, + _conn: &windmill_common::worker::Connection, + _job_succeeded: bool, +) { +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn sync_volumes_http_worker( + _volume_states: &[windmill_worker_volumes::VolumeState], + _volume_writable: &[bool], + _http: &windmill_common::worker::HttpClient, + _workspace_id: &str, + _job_id: uuid::Uuid, + _worker_name: &str, + _conn: &windmill_common::worker::Connection, + _job_succeeded: bool, +) { +} diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs new file mode 100644 index 0000000000..7c350ac92b --- /dev/null +++ b/backend/windmill-worker/src/wac_executor.rs @@ -0,0 +1,339 @@ +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use serde_json::Value; +use uuid::Uuid; + +use windmill_common::error::{self, Error}; +use windmill_common::DB; + +/// Checkpoint state persisted across workflow invocations. +#[derive(Debug, Serialize, Deserialize, Default, Clone)] +pub struct WacCheckpoint { + #[serde(default)] + pub source_hash: String, + #[serde(default)] + pub completed_steps: serde_json::Map, + #[serde(skip_serializing_if = "Option::is_none")] + pub pending_steps: Option, + #[serde(default)] + pub input_args: serde_json::Map, + /// Accumulated map of step_key → child job UUID across all dispatch rounds. + /// Unlike `pending_steps.job_ids` (cleared after completion), this persists + /// so the frontend can always resolve step keys to child job names. + #[serde(default)] + pub job_ids: serde_json::Map, + /// When set on a child job's checkpoint, indicates which step this child + /// should execute directly (instead of dispatching). + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub _executing_key: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct WacPendingSteps { + pub mode: String, + pub keys: Vec, + pub job_ids: serde_json::Map, +} + +/// Output from a single WAC invocation (parsed from result.json). +#[derive(Debug, Deserialize)] +#[serde(tag = "type")] +pub enum WacOutput { + #[serde(rename = "dispatch")] + Dispatch { mode: String, steps: Vec }, + #[serde(rename = "complete")] + Complete { result: Value }, + /// An inline step executed in the parent process — persist result to + /// checkpoint and re-run immediately (no child job, no suspend). + #[serde(rename = "inline_checkpoint")] + InlineCheckpoint { key: String, result: Value }, + /// Suspend the workflow waiting for an external approval event. + /// No child job is dispatched — the parent suspends directly and resumes + /// when a user hits the resume/cancel endpoint. + #[serde(rename = "approval")] + Approval { key: String, timeout: Option, form: Option }, + /// Server-side sleep — suspend the workflow for a duration without holding a worker. + #[serde(rename = "sleep")] + Sleep { key: String, seconds: u32 }, +} + +/// A step dispatched by the WAC SDK. +/// +/// `dispatch_type` determines how the child job is created: +/// - `"inline"` (default): re-runs the parent workflow with `_executing_key` set +/// - `"script"`: runs a separate Windmill script resolved from `script` path +/// - `"flow"`: runs a separate Windmill flow resolved from `script` path +#[derive(Debug, Deserialize, Clone)] +pub struct WacStepDispatch { + pub name: String, + pub script: String, + pub args: serde_json::Map, + pub key: String, + #[serde(default = "default_dispatch_type")] + pub dispatch_type: String, + // Per-task options forwarded to push() + #[serde(default)] + pub timeout: Option, + #[serde(default)] + pub tag: Option, + #[serde(default)] + pub cache_ttl: Option, + #[serde(default)] + pub priority: Option, + #[serde(default)] + pub concurrent_limit: Option, + #[serde(default)] + pub concurrency_key: Option, + #[serde(default)] + pub concurrency_time_window_s: Option, +} + +fn default_dispatch_type() -> String { + "inline".to_string() +} + +/// Load the WAC checkpoint from `v2_job_status.workflow_as_code_status._checkpoint`. +pub async fn load_checkpoint(db: &DB, job_id: &Uuid) -> error::Result { + let row: Option> = sqlx::query_scalar( + "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = $1", + ) + .bind(job_id) + .fetch_optional(db) + .await?; + + match row { + Some(Some(status)) => { + let checkpoint: WacCheckpoint = match serde_json::from_value(status) { + Ok(c) => c, + Err(e) => { + tracing::warn!( + job_id = %job_id, + error = %e, + "Failed to deserialize WAC checkpoint, resetting to empty" + ); + WacCheckpoint::default() + } + }; + Ok(checkpoint) + } + _ => Ok(WacCheckpoint::default()), + } +} + +/// Save the WAC checkpoint to `v2_job_status.workflow_as_code_status._checkpoint`. +/// The top level of workflow_as_code_status is reserved for per-child-job timeline data. +pub async fn save_checkpoint( + db: &DB, + job_id: &Uuid, + checkpoint: &WacCheckpoint, +) -> error::Result<()> { + let status_json = serde_json::to_value(checkpoint) + .map_err(|e| Error::InternalErr(format!("Failed to serialize checkpoint: {e}")))?; + + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(job_id) + .bind(&status_json) + .execute(db) + .await + .map_err(|e| Error::InternalErr(format!("Failed to save WAC checkpoint: {e}")))?; + + Ok(()) +} + +/// Parse the WAC result from result.json content. +pub fn parse_wac_output(result: &RawValue) -> error::Result { + serde_json::from_str(result.get()) + .map_err(|e| Error::InternalErr(format!("Failed to parse WAC output: {e}"))) +} + +/// Process a "dispatch" result: update checkpoint with pending steps info. +pub fn update_checkpoint_for_dispatch( + checkpoint: &mut WacCheckpoint, + steps: &[WacStepDispatch], + mode: &str, + job_ids: &[(String, Uuid)], +) { + let ids_map: serde_json::Map = job_ids + .iter() + .map(|(key, id)| (key.clone(), Value::String(id.to_string()))) + .collect(); + // Accumulate into persistent job_ids (survives pending_steps clearing) + for (k, v) in ids_map.iter() { + checkpoint.job_ids.insert(k.clone(), v.clone()); + } + let pending = WacPendingSteps { + mode: mode.to_string(), + keys: steps.iter().map(|s| s.key.clone()).collect(), + job_ids: ids_map, + }; + checkpoint.pending_steps = Some(pending); +} + +/// Process a completed child job result: add to checkpoint's completed_steps. +pub fn add_completed_step(checkpoint: &mut WacCheckpoint, step_key: &str, result: Value) { + checkpoint + .completed_steps + .insert(step_key.to_string(), result); + // If all pending steps are complete, clear pending + if let Some(ref pending) = checkpoint.pending_steps { + let all_done = pending + .keys + .iter() + .all(|k| checkpoint.completed_steps.contains_key(k)); + if all_done { + checkpoint.pending_steps = None; + } + } +} + +/// Check if all pending parallel steps are complete. +pub fn all_pending_complete(checkpoint: &WacCheckpoint) -> bool { + match &checkpoint.pending_steps { + None => true, + Some(pending) => pending + .keys + .iter() + .all(|k| checkpoint.completed_steps.contains_key(k)), + } +} + +/// If the checkpoint has a pending approval or sleep, inject the resume result +/// into `completed_steps` and save back to DB. Returns the (possibly modified) checkpoint. +/// +/// Called by both bun and python executors before writing checkpoint.json to disk. +pub async fn prepare_checkpoint_for_resume( + db: &DB, + job_id: &Uuid, + mut checkpoint: WacCheckpoint, +) -> error::Result { + let pending_mode = checkpoint.pending_steps.as_ref().map(|p| p.mode.as_str()); + + match pending_mode { + Some("approval") => { + let approval_key = checkpoint + .pending_steps + .as_ref() + .and_then(|p| p.keys.first().cloned()) + .unwrap_or_default(); + + let resume_row = sqlx::query_as::<_, (sqlx::types::Json>, Option, bool)>( + "SELECT value, approver, approved FROM resume_job WHERE job = $1 ORDER BY created_at ASC LIMIT 1", + ) + .bind(job_id) + .fetch_optional(db) + .await?; + + let approval_result = if let Some((value, approver, approved)) = resume_row { + serde_json::json!({ + "value": serde_json::from_str::(value.get()).unwrap_or(Value::Null), + "approver": approver.unwrap_or_else(|| "anonymous".to_string()), + "approved": approved, + }) + } else { + serde_json::json!({ + "value": null, + "approver": null, + "approved": false, + }) + }; + checkpoint + .completed_steps + .insert(approval_key.clone(), approval_result); + checkpoint.pending_steps = None; + save_checkpoint(db, job_id, &checkpoint).await?; + + tracing::info!( + job_id = %job_id, + approval_key = %approval_key, + "WAC v2 injected approval result into checkpoint" + ); + } + Some("sleep") => { + let sleep_key = checkpoint + .pending_steps + .as_ref() + .and_then(|p| p.keys.first().cloned()) + .unwrap_or_default(); + + checkpoint + .completed_steps + .insert(sleep_key.clone(), Value::Bool(true)); + checkpoint.pending_steps = None; + save_checkpoint(db, job_id, &checkpoint).await?; + + tracing::info!( + job_id = %job_id, + sleep_key = %sleep_key, + "WAC v2 resumed from sleep" + ); + } + _ => {} + } + + Ok(checkpoint) +} + +/// Detect WAC v2 patterns in TypeScript/Bun code. +/// Checks for `import ... from "windmill-client"` containing workflow/task, +/// skipping comment lines. +pub fn is_wac_v2_ts(code: &str) -> bool { + let mut has_wac_import = false; + let mut has_workflow = false; + let mut has_task = false; + for line in code.lines() { + let trimmed = line.trim(); + if trimmed.starts_with("//") { + continue; + } + if trimmed.contains("windmill-client") + && (trimmed.starts_with("import") || trimmed.starts_with("from")) + { + has_wac_import = true; + if trimmed.contains("workflow") { + has_workflow = true; + } + if trimmed.contains("task") { + has_task = true; + } + } + if trimmed.contains("export") && trimmed.contains("workflow(") { + has_workflow = true; + } + } + has_wac_import && has_workflow && has_task +} + +/// Detect WAC v2 patterns in Python code. +/// Checks for `@workflow` decorator and `@task` decorator with wmill import, +/// skipping comment lines. +pub fn is_wac_v2_py(code: &str) -> bool { + let mut has_wmill_import = false; + let mut has_workflow_decorator = false; + let mut has_task_decorator = false; + for line in code.lines() { + let trimmed = line.trim(); + if trimmed.starts_with('#') { + continue; + } + if trimmed.starts_with("import wmill") || trimmed.starts_with("from wmill") { + has_wmill_import = true; + } + if trimmed == "@workflow" || trimmed.starts_with("@workflow(") { + has_workflow_decorator = true; + } + if trimmed == "@task" || trimmed.starts_with("@task(") { + has_task_decorator = true; + } + } + has_wmill_import && has_workflow_decorator && has_task_decorator +} diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 49a1048afd..6d86ae2723 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -563,7 +563,16 @@ lazy_static::lazy_static! { pub static ref DOTNET_PATH: String = std::env::var("DOTNET_PATH").unwrap_or_else(|_| DOTNET_DEFAULT_PATH.to_string()); pub static ref NSJAIL_PATH: String = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string()); pub static ref PATH_ENV: String = std::env::var("PATH").unwrap_or_else(|_| String::new()); - pub static ref HOME_ENV: String = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()); + pub static ref HOME_ENV: String = { + #[cfg(not(windows))] + { std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()) } + #[cfg(windows)] + { + std::env::var("HOME") + .or_else(|_| std::env::var("USERPROFILE")) + .unwrap_or_else(|_| std::env::temp_dir().to_string_lossy().to_string()) + } + }; pub static ref GIT_PATH: String = std::env::var("GIT_PATH").unwrap_or_else(|_| "/usr/bin/git".to_string()); pub static ref NODE_PATH: Option = std::env::var("NODE_PATH").ok(); @@ -731,26 +740,69 @@ pub async fn is_otel_tracing_proxy_enabled_for_lang(lang: &ScriptLang) -> bool { } } +/// Get OTEL trace context environment variables for a job (TRACEPARENT, OTEL_TRACE_ID, OTEL_SPAN_ID). +/// Returns an empty vec when OTEL tracing is not enabled or on non-enterprise builds. +pub fn get_otel_context_envs(job_id: &uuid::Uuid) -> Vec<(&'static str, String)> { + #[cfg(all(feature = "private", feature = "enterprise"))] + if windmill_common::OTEL_TRACING_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { + let trace_id = format!("{:032x}", job_id.as_u128()); + let span_id = format!("{:016x}", job_id.as_u64_pair().1); + let traceparent = format!("00-{}-{}-01", trace_id, span_id); + return vec![ + ("TRACEPARENT", traceparent), + ("OTEL_TRACE_ID", trace_id), + ("OTEL_SPAN_ID", span_id), + ]; + } + let _ = job_id; + vec![] +} + /// Get proxy environment variables for job execution for a specific language. /// When OTEL tracing proxy is enabled for this language, routes all traffic through the proxy. /// Otherwise, uses the standard HTTP_PROXY/HTTPS_PROXY from environment. pub async fn get_proxy_envs_for_lang( lang: &ScriptLang, + job_id: &uuid::Uuid, + w_id: &str, + conn: &Connection, ) -> anyhow::Result> { + #[allow(unused_mut)] + let mut envs; #[cfg(all(feature = "private", feature = "enterprise"))] if is_otel_tracing_proxy_enabled_for_lang(lang).await { - return get_otel_tracing_proxy_envs().await; + envs = get_otel_tracing_proxy_envs(job_id, w_id, conn).await?; + } else { + envs = PROXY_ENVS.clone(); } - let _ = lang; - Ok(PROXY_ENVS.clone()) + #[cfg(not(all(feature = "private", feature = "enterprise")))] + { + let _ = (lang, w_id, conn); + envs = PROXY_ENVS.clone(); + } + envs.extend(get_otel_context_envs(job_id)); + Ok(envs) } #[cfg(all(feature = "private", feature = "enterprise"))] -async fn get_otel_tracing_proxy_envs() -> anyhow::Result> { - let port = crate::otel_tracing_proxy_ee::TRACING_PROXY_PORT +async fn get_otel_tracing_proxy_envs( + job_id: &uuid::Uuid, + w_id: &str, + conn: &Connection, +) -> anyhow::Result> { + let port = match *crate::otel_tracing_proxy_ee::TRACING_PROXY_PORT .read() .await - .ok_or_else(|| anyhow::anyhow!("OTEL tracing proxy port not initialized"))?; + { + Some(p) => p, + None => { + let reason = "OTEL tracing proxy is enabled but not available (not initialized yet, or NUM_WORKERS > 1). \ + This job's HTTP requests will not be traced."; + tracing::warn!("{}", reason); + append_logs(job_id, w_id, format!("\n[warning] {reason}\n"), conn).await; + return Ok(PROXY_ENVS.clone()); + } + }; let proxy_url = format!("http://127.0.0.1:{}", port); Ok(vec![ ("HTTP_PROXY", proxy_url.clone()), @@ -3476,6 +3528,13 @@ pub async fn handle_queued_job( { return Ok(false); } + if result + .as_ref() + .is_err_and(|err| matches!(err, &Error::WacSuspended(_))) + { + // WAC v2 job suspended while waiting for child jobs — don't complete it + return Ok(true); + } process_result( cjob, result.map(|x| Arc::new(x)), @@ -3874,7 +3933,7 @@ pub async fn run_language_executor( run_inline: bool, ) -> error::Result> { if language == Some(ScriptLang::Postgresql) { - return do_postgresql( + return Box::pin(do_postgresql( job, &client, &code, @@ -3886,7 +3945,7 @@ pub async fn run_language_executor( occupancy_metrics, parent_runnable_path, run_inline, - ) + )) .await; } else if language == Some(ScriptLang::Mysql) { #[cfg(not(feature = "mysql"))] @@ -3901,7 +3960,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_mysql( + return Box::pin(do_mysql( job, &client, &code, @@ -3912,7 +3971,7 @@ pub async fn run_language_executor( column_order, occupancy_metrics, parent_runnable_path, - ) + )) .await; } } else if language == Some(ScriptLang::Bigquery) { @@ -3938,7 +3997,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_bigquery( + return Box::pin(do_bigquery( job, &client, &code, @@ -3949,7 +4008,7 @@ pub async fn run_language_executor( column_order, occupancy_metrics, parent_runnable_path, - ) + )) .await; } } else if language == Some(ScriptLang::Snowflake) { @@ -3967,7 +4026,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_snowflake( + return Box::pin(do_snowflake( job, &client, &code, @@ -3978,7 +4037,7 @@ pub async fn run_language_executor( column_order, occupancy_metrics, parent_runnable_path, - ) + )) .await; } } else if language == Some(ScriptLang::Mssql) { @@ -4004,7 +4063,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_mssql( + return Box::pin(do_mssql( job, &client, &code, @@ -4015,7 +4074,7 @@ pub async fn run_language_executor( occupancy_metrics, job_dir, parent_runnable_path, - ) + )) .await; } } else if language == Some(ScriptLang::OracleDB) { @@ -4041,7 +4100,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_oracledb( + return Box::pin(do_oracledb( job, &client, &code, @@ -4052,7 +4111,7 @@ pub async fn run_language_executor( column_order, occupancy_metrics, parent_runnable_path, - ) + )) .await; } } else if language == Some(ScriptLang::DuckDb) { @@ -4066,7 +4125,7 @@ pub async fn run_language_executor( #[cfg(feature = "duckdb")] { - return do_duckdb( + return Box::pin(do_duckdb( job, &client, &code, @@ -4078,7 +4137,7 @@ pub async fn run_language_executor( occupancy_metrics, parent_runnable_path, run_inline, - ) + )) .await; } } else if language == Some(ScriptLang::Graphql) { @@ -4087,7 +4146,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_graphql( + return Box::pin(do_graphql( job, &client, &code, @@ -4096,7 +4155,7 @@ pub async fn run_language_executor( canceled_by, worker_name, occupancy_metrics, - ) + )) .await; } else if language == Some(ScriptLang::Nativets) { if run_inline { @@ -4123,7 +4182,7 @@ pub async fn run_language_executor( .collect::>() .join("\n")); - let result = do_nativets( + let result = Box::pin(do_nativets( job, &client, env_code, @@ -4134,7 +4193,7 @@ pub async fn run_language_executor( worker_name, occupancy_metrics, has_stream, - ) + )) .await?; return Ok(result); } @@ -4152,7 +4211,8 @@ pub async fn run_language_executor( job.id ); - let shared_mount = if job.same_worker && job.script_lang != Some(ScriptLang::Deno) { + #[allow(unused_mut)] + let mut shared_mount = if job.same_worker && job.script_lang != Some(ScriptLang::Deno) { let folder = if job.script_lang == Some(ScriptLang::Go) { "/go" } else { @@ -4174,7 +4234,8 @@ mount {{ // println!("handle lang job {:?}", SystemTime::now()); - let envs = build_envs(envs.as_ref())?; + #[allow(unused_mut)] + let mut envs = build_envs(envs.as_ref())?; let Some(language) = language else { return Err(Error::ExecutionErr( @@ -4210,6 +4271,106 @@ mount {{ } } + // Volume mount setup (requires workspace S3 storage; CE has file count/size limits) + #[cfg(feature = "parquet")] + let volume_mounts = { + let comment_prefix = match language { + ScriptLang::Python3 + | ScriptLang::Bash + | ScriptLang::Powershell + | ScriptLang::Ansible + | ScriptLang::Ruby => "#", + ScriptLang::Deno + | ScriptLang::Bun + | ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Go => "//", + _ => "", + }; + let raw_mounts = windmill_worker_volumes::parse_volume_annotations(&code, comment_prefix); + let args_ref = job.args.as_ref().map(|a| &**a); + let mut interpolated = Vec::new(); + for mut v in raw_mounts { + v.name = windmill_worker_volumes::interpolate_volume_name( + &v.name, + args_ref, + &job.workspace_id, + ); + if let Err(e) = windmill_worker_volumes::validate_volume_name(&v.name) { + return Err(Error::ExecutionErr(e)); + } + if let Err(e) = windmill_worker_volumes::validate_volume_target(&v.target) { + return Err(Error::ExecutionErr(e)); + } + interpolated.push(v); + } + if let Err(e) = windmill_worker_volumes::validate_volume_mounts(&interpolated) { + return Err(Error::ExecutionErr(e)); + } + interpolated + }; + + #[cfg(feature = "parquet")] + let mut volume_setup = crate::volume_oss::VolumeSetupResult { + states: Vec::new(), + writable: Vec::new(), + client: None, + lease_renewal: crate::volume_oss::LeaseRenewalGuard(None), + }; + + #[cfg(feature = "parquet")] + if !volume_mounts.is_empty() { + let vol_summary: Vec = volume_mounts + .iter() + .map(|v| format!("'{}' -> {}", v.name, v.target)) + .collect(); + append_logs( + &job.id, + &job.workspace_id, + format!( + "\n--- VOLUME MOUNTS ---\nPulling {} volume(s): {}\n", + volume_mounts.len(), + vol_summary.join(", "), + ), + conn, + ) + .await; + + if let Connection::Sql(db) = conn { + volume_setup = crate::volume_oss::setup_volumes_sql_worker( + &volume_mounts, + db, + &job.workspace_id, + job.id, + &job.permissioned_as, + worker_name, + job_dir, + client, + conn, + language, + &mut envs, + &mut shared_mount, + ) + .await?; + } else if let Connection::Http(http) = conn { + volume_setup = crate::volume_oss::setup_volumes_http_worker( + &volume_mounts, + http, + &job.workspace_id, + job.id, + &job.permissioned_as, + &job.canceled_by, + worker_name, + job_dir, + conn, + language, + &mut envs, + &mut shared_mount, + ) + .await?; + } + } + // Box::pin all language handlers to prevent large match enum on stack let result: error::Result> = match language { ScriptLang::Python3 => { @@ -4621,6 +4782,62 @@ mount {{ // for related places search: ADD_NEW_LANG _ => panic!("unreachable, language is not supported: {language:#?}"), }; + // Volume sync-back and lease release + #[cfg(feature = "parquet")] + if !volume_setup.states.is_empty() { + // Stop lease renewal before sync-back + volume_setup.lease_renewal.0.take().map(|h| h.abort()); + + if let Some(ref vol_client) = volume_setup.client { + if let Connection::Sql(db) = conn { + crate::volume_oss::sync_volumes_sql_worker( + &volume_setup.states, + &volume_setup.writable, + vol_client, + db, + &job.workspace_id, + job.id, + worker_name, + conn, + result.is_ok(), + ) + .await; + } + } + + if let Connection::Http(http) = conn { + crate::volume_oss::sync_volumes_http_worker( + &volume_setup.states, + &volume_setup.writable, + http, + &job.workspace_id, + job.id, + worker_name, + conn, + result.is_ok(), + ) + .await; + } + + // Clean up absolute-path symlinks created by setup_volume_mount_paths + if !is_sandboxing_enabled() { + #[allow(unused_variables)] // state is only used on unix + for state in &volume_setup.states { + #[cfg(unix)] + if state.mount.target.starts_with('/') { + let target_path = std::path::Path::new(&state.mount.target); + if target_path + .symlink_metadata() + .map(|m| m.file_type().is_symlink()) + .unwrap_or(false) + { + std::fs::remove_file(target_path).ok(); + } + } + } + } + } + tracing::info!( workspace_id = %job.workspace_id, is_ok = result.is_ok(), diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 33dafbb482..24dcfb9b9a 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -865,7 +865,7 @@ pub async fn update_flow_status_after_job_completion_internal( .and_then(|x| x.stop_after_all_iters_if.as_ref()) { let args = from_result_to_args(args.as_ref().await.get_ref())?; - evaluate_stop_after_all_iters_if( + if let Err(e) = evaluate_stop_after_all_iters_if( db, stop_after_all_iters_if, module_status, @@ -879,7 +879,16 @@ pub async fn update_flow_status_after_job_completion_internal( flow, &old_status, ) - .await?; + .await + { + tracing::error!("error evaluating stop_after_all_iters_if: {e:#}"); + stop_early = true; + skip_if_stop_early = false; + stop_early_err_msg = Some(format!( + "Error evaluating stop_after_all_iters_if expression `{}`: {e:#}", + stop_after_all_iters_if.expr + )); + } } let new_status = if @@ -1074,7 +1083,7 @@ pub async fn update_flow_status_after_job_completion_internal( { let args = from_result_to_args(args.as_ref().await.get_ref())?; - evaluate_stop_after_all_iters_if( + if let Err(e) = evaluate_stop_after_all_iters_if( db, stop_after_all_iters_if, module_status, @@ -1088,7 +1097,15 @@ pub async fn update_flow_status_after_job_completion_internal( flow, &old_status, ) - .await?; + .await + { + stop_early = true; + skip_if_stop_early = false; + stop_early_err_msg = Some(format!( + "Error evaluating stop_after_all_iters_if expression `{}`: {e:#}", + stop_after_all_iters_if.expr + )); + } } } @@ -1705,8 +1722,8 @@ pub async fn update_flow_status_after_job_completion_internal( chat_ai_info.conversation_id, ) .await?; - let duration = if success { - let (_, duration) = add_completed_job( + let (duration, wac_job_ids) = if success { + let (_, duration, wac_job_ids) = add_completed_job( db, &cflow_job, true, @@ -1720,9 +1737,9 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - duration + (duration, wac_job_ids) } else { - let (_, duration) = add_completed_job( + let (_, duration, wac_job_ids) = add_completed_job( db, &cflow_job, false, @@ -1740,11 +1757,30 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - duration + (duration, wac_job_ids) }; flow_job_duration = flow_job .started_at .map(|x| FlowJobDuration { started_at: x, duration_ms: duration }); + + // If this flow is a WAC child (not a flow step, has parent), + // notify the WAC parent of completion. + if !flow_job.is_flow_step() { + if let Some(parent_job) = flow_job.parent_job { + if let Some(job_ids) = wac_job_ids { + let _ = crate::result_processor::handle_wac_child_completion( + db, + &flow_job.id, + parent_job, + &flow_job.workspace_id, + nresult.clone(), + success, + job_ids, + ) + .await; + } + } + } } true } else { diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index cc1417c4b3..121d5b6c6d 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.649.0"; +export const VERSION = "v1.653.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/branchone-tab.png b/branchone-tab.png new file mode 100644 index 0000000000..e9bf7cc602 Binary files /dev/null and b/branchone-tab.png differ diff --git a/centered-summary.png b/centered-summary.png new file mode 100644 index 0000000000..ca1b7d8c55 Binary files /dev/null and b/centered-summary.png differ diff --git a/cli/bootstrap/script_bootstrap.ts b/cli/bootstrap/script_bootstrap.ts index 5335e1741c..89d98a1927 100644 --- a/cli/bootstrap/script_bootstrap.ts +++ b/cli/bootstrap/script_bootstrap.ts @@ -39,7 +39,10 @@ export const scriptBootstrapCode = { } `, - bun: `export async function main() { + bun: `// there are multiple modes to add as header: //nobundling //native //npm //nodejs +// https://www.windmill.dev/docs/getting_started/scripts_quickstart/typescript#modes + +export async function main() { return "Hello world"; } `, diff --git a/cli/dev.nu b/cli/dev.nu new file mode 100644 index 0000000000..91a0faa8e3 --- /dev/null +++ b/cli/dev.nu @@ -0,0 +1,59 @@ +#!/usr/bin/env nu + +let cli_cache = "/tmp/windmill/cache_nomount/bun/" +let bundle_cache = "/tmp/windmill/cache/bun/" + +# Clean CLI package cache +def "main clean" [] { + rm -rf ($cli_cache ++ "windmill-cli@*") + rm -rf ($cli_cache ++ "windmill-cli/") + print "Cleaned CLI cache" +} + +# Clear bundle cache (forces hub scripts to re-bundle) +def "main clear-bundles" [] { + rm -rf ($bundle_cache ++ "*") + print "Cleared bundle cache" +} + +# Patch CLI cache with local build +def "main patch" [] { + print "Patching CLI cache..." + + let versions = (ls $cli_cache | where name =~ "windmill-cli@" | get name) + + if ($versions | is-empty) { + print "No CLI versions found in cache" + return + } + + for path in $versions { + rm -rf ($path ++ "/esm") + ^cp -r npm/esm ($path ++ "/esm") + ^cp npm/package.json ($path ++ "/package.json") + print $"Patched ($path | path basename)" + } + + print "Done!" +} + +# Build CLI, patch cache, and clear bundles +def main [ + --patch(-p) # Only patch existing cache (skip build) + --clean(-c) # Clean CLI cache first +] { + if $clean { + main clean + } + + if $patch { + main patch + } else { + print "Building CLI..." + bun run build + main patch + } + + # Always clear bundle cache so hub scripts use patched CLI + main clear-bundles +} diff --git a/cli/src/commands/gitsync-settings/converter.ts b/cli/src/commands/gitsync-settings/converter.ts index 89600fd5d1..6861603bb4 100644 --- a/cli/src/commands/gitsync-settings/converter.ts +++ b/cli/src/commands/gitsync-settings/converter.ts @@ -33,6 +33,7 @@ export class GitSyncSettingsConverter { includeGroups: includeTypes.includes("group"), includeSettings: includeTypes.includes("settings"), includeKey: includeTypes.includes("key"), + skipWorkspaceDependencies: !includeTypes.includes("workspacedependencies"), }; // Only include extraIncludes if it has content @@ -61,6 +62,7 @@ export class GitSyncSettingsConverter { if (opts.includeGroups) includeTypes.push("group"); if (opts.includeSettings) includeTypes.push("settings"); if (opts.includeKey) includeTypes.push("key"); + if (!opts.skipWorkspaceDependencies) includeTypes.push("workspacedependencies"); const result: BackendGitSyncSettings = { include_path: opts.includes || [], @@ -99,6 +101,7 @@ export class GitSyncSettingsConverter { includeGroups: opts.includeGroups ?? false, includeSettings: opts.includeSettings ?? false, includeKey: opts.includeKey ?? false, + skipWorkspaceDependencies: opts.skipWorkspaceDependencies ?? false, }; } @@ -122,6 +125,7 @@ export class GitSyncSettingsConverter { includeGroups: opts.includeGroups, includeSettings: opts.includeSettings, includeKey: opts.includeKey, + skipWorkspaceDependencies: opts.skipWorkspaceDependencies, }; } diff --git a/cli/src/commands/gitsync-settings/types.ts b/cli/src/commands/gitsync-settings/types.ts index 36b51aedbc..7bebb1f1a1 100644 --- a/cli/src/commands/gitsync-settings/types.ts +++ b/cli/src/commands/gitsync-settings/types.ts @@ -37,6 +37,7 @@ export const GIT_SYNC_FIELDS = [ "includeGroups", "includeSettings", "includeKey", + "skipWorkspaceDependencies", ] as const; export type GitSyncField = typeof GIT_SYNC_FIELDS[number]; @@ -57,6 +58,7 @@ export const INCLUDE_TYPE_MAPPINGS = { group: "includeGroups", settings: "includeSettings", key: "includeKey", + workspacedependencies: "skipWorkspaceDependencies", } as const; // Write mode for branch-based configuration diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 6d4a359600..7d48a14848 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -2626,6 +2626,7 @@ export async function push( let [_basePath, changes] = queue.shift()!; const promise = (async () => { const alreadySynced: string[] = []; + const deletedVarsResPaths: string[] = []; const isRawApp = isRawAppFile(changes[0].path); if (isRawApp) { const deleteRawApp = changes.find( @@ -2870,12 +2871,23 @@ export async function push( name: change.path.split(SEP)[1], }); break; - case "resource": - await wmill.deleteResource({ - workspace: workspaceId, - path: removeSuffix(target, ".resource.json"), - }); + case "resource": { + const resourcePath = removeSuffix(target, ".resource.json"); + try { + await wmill.deleteResource({ + workspace: workspaceId, + path: resourcePath, + }); + } catch (e: any) { + if (e?.status === 404 && deletedVarsResPaths.includes(resourcePath)) { + log.debug(`Resource ${resourcePath} already deleted by linked variable`); + } else { + throw e; + } + } + deletedVarsResPaths.push(resourcePath); break; + } case "resource-type": await wmill.deleteResourceType({ workspace: workspaceId, @@ -3012,12 +3024,23 @@ export async function push( }); break; } - case "variable": - await wmill.deleteVariable({ - workspace: workspaceId, - path: removeSuffix(target, ".variable.json"), - }); + case "variable": { + const variablePath = removeSuffix(target, ".variable.json"); + try { + await wmill.deleteVariable({ + workspace: workspaceId, + path: variablePath, + }); + } catch (e: any) { + if (e?.status === 404 && deletedVarsResPaths.includes(variablePath)) { + log.debug(`Variable ${variablePath} already deleted by linked resource`); + } else { + throw e; + } + } + deletedVarsResPaths.push(variablePath); break; + } case "user": { const users = await wmill.listUsers({ workspace: workspaceId, diff --git a/cli/src/commands/workspace/workspace.ts b/cli/src/commands/workspace/workspace.ts index b70469073c..6124bd1e1c 100644 --- a/cli/src/commands/workspace/workspace.ts +++ b/cli/src/commands/workspace/workspace.ts @@ -408,7 +408,8 @@ async function remove(_opts: GlobalOptions, name: string) { async function whoami(_opts: GlobalOptions) { await requireLogin(_opts); - log.info(await wmill.globalWhoami()); + const whoamiInfo = await wmill.globalWhoami(); + log.info(JSON.stringify(whoamiInfo, null, 2)); const activeName = await getActiveWorkspaceName(_opts); log.info("Active: " + colors.green.bold(activeName || "none")); } diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index 3ffddc4837..0f7acce4a9 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -53,6 +53,9 @@ export interface SimplifiedSettings { mute_critical_alerts?: boolean; color?: string; operator_settings?: any; + slack_team_id?: string; + slack_name?: string; + slack_command_script?: string; } // Legacy settings interface for reading old settings.yaml files @@ -77,6 +80,9 @@ interface LegacySimplifiedSettings { mute_critical_alerts?: boolean; color?: string; operator_settings?: any; + slack_team_id?: string; + slack_name?: string; + slack_command_script?: string; } // Helper to convert legacy flat settings to new grouped format @@ -94,6 +100,9 @@ export function migrateToGroupedFormat(settings: any): SimplifiedSettings { if (settings.mute_critical_alerts !== undefined) result.mute_critical_alerts = settings.mute_critical_alerts; if (settings.color !== undefined) result.color = settings.color; if (settings.operator_settings !== undefined) result.operator_settings = settings.operator_settings; + if (settings.slack_team_id !== undefined) result.slack_team_id = settings.slack_team_id; + if (settings.slack_name !== undefined) result.slack_name = settings.slack_name; + if (settings.slack_command_script !== undefined) result.slack_command_script = settings.slack_command_script; // Handle auto_invite: check if already grouped or needs migration if (settings.auto_invite && typeof settings.auto_invite === "object") { @@ -183,12 +192,18 @@ export async function pushWorkspaceSettings( mute_critical_alerts: remoteSettings.mute_critical_alerts, color: remoteSettings.color, operator_settings: remoteSettings.operator_settings, + slack_team_id: remoteSettings.slack_team_id, + slack_name: remoteSettings.slack_name, + slack_command_script: remoteSettings.slack_command_script, }; } catch (err) { throw new Error(`Failed to get workspace settings: ${err}`); } - if (isSuperset(localSettings, settings)) { + // Exclude read-only fields from comparison (slack_team_id and slack_name are set via OAuth only) + const { slack_team_id: _lst, slack_name: _lsn, ...comparableLocal } = localSettings; + const { slack_team_id: _rst, slack_name: _rsn, ...comparableRemote } = settings; + if (isSuperset(comparableLocal, comparableRemote)) { log.debug(`Workspace settings are up to date`); return; } @@ -366,6 +381,16 @@ export async function pushWorkspaceSettings( requestBody: localSettings.operator_settings, }); } + + if (localSettings.slack_command_script != settings.slack_command_script) { + log.debug(`Updating slack command script...`); + await wmill.editSlackCommand({ + workspace, + requestBody: { + slack_command_script: localSettings.slack_command_script, + }, + }); + } } export async function pushWorkspaceKey( diff --git a/cli/src/main.ts b/cli/src/main.ts index b584c540d8..6ca14e9975 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -67,7 +67,7 @@ export { workspaceAdd, }; -export const VERSION = "1.649.0"; +export const VERSION = "1.653.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/cli/test/gitsync_converter_unit.test.ts b/cli/test/gitsync_converter_unit.test.ts new file mode 100644 index 0000000000..e84c2ffa11 --- /dev/null +++ b/cli/test/gitsync_converter_unit.test.ts @@ -0,0 +1,227 @@ +/** + * Unit tests for GitSyncSettingsConverter. + * Tests conversion between backend format (include_type array) and + * SyncOptions format (boolean flags like skipWorkspaceDependencies). + */ + +import { expect, test, describe } from "bun:test"; +import { GitSyncSettingsConverter } from "../src/commands/gitsync-settings/converter.ts"; + +// ============================================================================= +// fromBackendFormat - converts backend include_type array to SyncOptions +// ============================================================================= + +describe("GitSyncSettingsConverter.fromBackendFormat", () => { + test("converts workspacedependencies in include_type to skipWorkspaceDependencies: false", () => { + const backend = { + include_path: ["f/**"], + include_type: ["script", "flow", "workspacedependencies"], + }; + const result = GitSyncSettingsConverter.fromBackendFormat(backend); + expect(result.skipWorkspaceDependencies).toBe(false); + }); + + test("sets skipWorkspaceDependencies: true when workspacedependencies is absent", () => { + const backend = { + include_path: ["f/**"], + include_type: ["script", "flow"], + }; + const result = GitSyncSettingsConverter.fromBackendFormat(backend); + expect(result.skipWorkspaceDependencies).toBe(true); + }); + + test("handles empty include_type array", () => { + const backend = { + include_path: [], + include_type: [], + }; + const result = GitSyncSettingsConverter.fromBackendFormat(backend); + expect(result.skipWorkspaceDependencies).toBe(true); + expect(result.skipScripts).toBe(true); + expect(result.skipFlows).toBe(true); + }); + + test("converts all standard types correctly", () => { + const backend = { + include_path: ["f/**"], + include_type: ["script", "flow", "app", "folder", "variable", "resource", "resourcetype", "secret", "schedule", "trigger", "user", "group", "settings", "key", "workspacedependencies"], + }; + const result = GitSyncSettingsConverter.fromBackendFormat(backend); + + expect(result.skipScripts).toBe(false); + expect(result.skipFlows).toBe(false); + expect(result.skipApps).toBe(false); + expect(result.skipFolders).toBe(false); + expect(result.skipVariables).toBe(false); + expect(result.skipResources).toBe(false); + expect(result.skipResourceTypes).toBe(false); + expect(result.skipSecrets).toBe(false); + expect(result.includeSchedules).toBe(true); + expect(result.includeTriggers).toBe(true); + expect(result.includeUsers).toBe(true); + expect(result.includeGroups).toBe(true); + expect(result.includeSettings).toBe(true); + expect(result.includeKey).toBe(true); + expect(result.skipWorkspaceDependencies).toBe(false); + }); +}); + +// ============================================================================= +// toBackendFormat - converts SyncOptions to backend include_type array +// ============================================================================= + +describe("GitSyncSettingsConverter.toBackendFormat", () => { + test("adds workspacedependencies when skipWorkspaceDependencies is false", () => { + const opts = { + includes: ["f/**"], + skipWorkspaceDependencies: false, + }; + const result = GitSyncSettingsConverter.toBackendFormat(opts); + expect(result.include_type).toContain("workspacedependencies"); + }); + + test("does not add workspacedependencies when skipWorkspaceDependencies is true", () => { + const opts = { + includes: ["f/**"], + skipWorkspaceDependencies: true, + }; + const result = GitSyncSettingsConverter.toBackendFormat(opts); + expect(result.include_type).not.toContain("workspacedependencies"); + }); + + test("adds workspacedependencies when skipWorkspaceDependencies is undefined (defaults to false)", () => { + const opts = { + includes: ["f/**"], + // skipWorkspaceDependencies not set + }; + const normalized = GitSyncSettingsConverter.normalize(opts); + const result = GitSyncSettingsConverter.toBackendFormat(normalized); + expect(result.include_type).toContain("workspacedependencies"); + }); + + test("converts all boolean flags to include_type correctly", () => { + const opts = { + includes: ["f/**"], + skipScripts: false, + skipFlows: false, + skipApps: false, + skipFolders: false, + skipVariables: false, + skipResources: false, + skipResourceTypes: false, + skipSecrets: false, + includeSchedules: true, + includeTriggers: true, + includeUsers: true, + includeGroups: true, + includeSettings: true, + includeKey: true, + skipWorkspaceDependencies: false, + }; + const result = GitSyncSettingsConverter.toBackendFormat(opts); + + expect(result.include_type).toContain("script"); + expect(result.include_type).toContain("flow"); + expect(result.include_type).toContain("app"); + expect(result.include_type).toContain("folder"); + expect(result.include_type).toContain("variable"); + expect(result.include_type).toContain("resource"); + expect(result.include_type).toContain("resourcetype"); + expect(result.include_type).toContain("secret"); + expect(result.include_type).toContain("schedule"); + expect(result.include_type).toContain("trigger"); + expect(result.include_type).toContain("user"); + expect(result.include_type).toContain("group"); + expect(result.include_type).toContain("settings"); + expect(result.include_type).toContain("key"); + expect(result.include_type).toContain("workspacedependencies"); + }); +}); + +// ============================================================================= +// normalize - applies defaults for undefined fields +// ============================================================================= + +describe("GitSyncSettingsConverter.normalize", () => { + test("defaults skipWorkspaceDependencies to false", () => { + const opts = { includes: ["f/**"] }; + const result = GitSyncSettingsConverter.normalize(opts); + expect(result.skipWorkspaceDependencies).toBe(false); + }); + + test("preserves explicit skipWorkspaceDependencies: true", () => { + const opts = { includes: ["f/**"], skipWorkspaceDependencies: true }; + const result = GitSyncSettingsConverter.normalize(opts); + expect(result.skipWorkspaceDependencies).toBe(true); + }); + + test("preserves explicit skipWorkspaceDependencies: false", () => { + const opts = { includes: ["f/**"], skipWorkspaceDependencies: false }; + const result = GitSyncSettingsConverter.normalize(opts); + expect(result.skipWorkspaceDependencies).toBe(false); + }); +}); + +// ============================================================================= +// Round-trip conversion tests +// ============================================================================= + +describe("GitSyncSettingsConverter round-trip", () => { + test("backend -> SyncOptions -> backend preserves workspacedependencies", () => { + const original = { + include_path: ["f/**"], + include_type: ["script", "flow", "workspacedependencies"], + }; + + const syncOpts = GitSyncSettingsConverter.fromBackendFormat(original); + const backAgain = GitSyncSettingsConverter.toBackendFormat(syncOpts); + + expect(backAgain.include_type).toContain("workspacedependencies"); + expect(backAgain.include_type).toContain("script"); + expect(backAgain.include_type).toContain("flow"); + }); + + test("backend without workspacedependencies -> SyncOptions -> backend still excludes it", () => { + const original = { + include_path: ["f/**"], + include_type: ["script", "flow"], + }; + + const syncOpts = GitSyncSettingsConverter.fromBackendFormat(original); + const backAgain = GitSyncSettingsConverter.toBackendFormat(syncOpts); + + expect(backAgain.include_type).not.toContain("workspacedependencies"); + expect(backAgain.include_type).toContain("script"); + expect(backAgain.include_type).toContain("flow"); + }); + + test("SyncOptions with defaults -> backend includes workspacedependencies", () => { + const opts = { + includes: ["f/**"], + skipScripts: false, + skipFlows: false, + // skipWorkspaceDependencies not set - should default to false + }; + + const normalized = GitSyncSettingsConverter.normalize(opts); + const backend = GitSyncSettingsConverter.toBackendFormat(normalized); + + expect(backend.include_type).toContain("workspacedependencies"); + }); +}); + +// ============================================================================= +// extractGitSyncFields +// ============================================================================= + +describe("GitSyncSettingsConverter.extractGitSyncFields", () => { + test("includes skipWorkspaceDependencies in extracted fields", () => { + const opts = { + includes: ["f/**"], + skipWorkspaceDependencies: true, + someOtherField: "ignored", + }; + const result = GitSyncSettingsConverter.extractGitSyncFields(opts); + expect(result.skipWorkspaceDependencies).toBe(true); + }); +}); diff --git a/cli/test/settings_unit.test.ts b/cli/test/settings_unit.test.ts index 2d6a5249ef..014456cfc0 100644 --- a/cli/test/settings_unit.test.ts +++ b/cli/test/settings_unit.test.ts @@ -193,5 +193,21 @@ describe("migrateToGroupedFormat", () => { expect("webhook" in result).toBe(false); expect("deploy_to" in result).toBe(false); expect("color" in result).toBe(false); + expect("slack_team_id" in result).toBe(false); + expect("slack_name" in result).toBe(false); + expect("slack_command_script" in result).toBe(false); + }); + + test("copies slack fields through", () => { + const settings = { + name: "ws", + slack_team_id: "T12345", + slack_name: "my-team", + slack_command_script: "u/admin/slack_handler", + }; + const result = migrateToGroupedFormat(settings); + expect(result.slack_team_id).toBe("T12345"); + expect(result.slack_name).toBe("my-team"); + expect(result.slack_command_script).toBe("u/admin/slack_handler"); }); }); diff --git a/collapsed-forloop-full.png b/collapsed-forloop-full.png new file mode 100644 index 0000000000..8abad55080 Binary files /dev/null and b/collapsed-forloop-full.png differ diff --git a/collapsed-forloop-tab.png b/collapsed-forloop-tab.png new file mode 100644 index 0000000000..8abad55080 Binary files /dev/null and b/collapsed-forloop-tab.png differ diff --git a/collapsed-with-note.png b/collapsed-with-note.png new file mode 100644 index 0000000000..a8dd6e0033 Binary files /dev/null and b/collapsed-with-note.png differ diff --git a/current-state.png b/current-state.png new file mode 100644 index 0000000000..bdfa8106b0 Binary files /dev/null and b/current-state.png differ diff --git a/current-state2.png b/current-state2.png new file mode 100644 index 0000000000..a98edbdd89 Binary files /dev/null and b/current-state2.png differ diff --git a/current-tab-state.png b/current-tab-state.png new file mode 100644 index 0000000000..c71de9d07c Binary files /dev/null and b/current-tab-state.png differ diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index 450dd68399..279c79b780 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -61,6 +61,11 @@ RUN ln -s /usr/bin/bun /usr/bin/node \ && bun install -g windmill-cli \ && ln -s $(bun pm bin -g)/wmill /usr/bin/wmill +# Install Claude Code CLI (used by claude sandbox scripts) +# Copy to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root) +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && cp /root/.local/share/claude/versions/* /usr/bin/claude + # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 7cc4dafa05..88c16aaac0 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -61,6 +61,11 @@ RUN ln -s /usr/bin/bun /usr/bin/node \ && bun install -g windmill-cli \ && ln -s $(bun pm bin -g)/wmill /usr/bin/wmill +# Install Claude Code CLI (used by claude sandbox scripts) +# Copy to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root) +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && cp /root/.local/share/claude/versions/* /usr/bin/claude + # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/flow-current-state.png b/flow-current-state.png new file mode 100644 index 0000000000..f80e5e2334 Binary files /dev/null and b/flow-current-state.png differ diff --git a/flow-with-group.png b/flow-with-group.png new file mode 100644 index 0000000000..22c5ca5738 Binary files /dev/null and b/flow-with-group.png differ diff --git a/forloop-region.png b/forloop-region.png new file mode 100644 index 0000000000..e7028ea503 Binary files /dev/null and b/forloop-region.png differ diff --git a/forloop-tab-v2.png b/forloop-tab-v2.png new file mode 100644 index 0000000000..03d88fa4ad Binary files /dev/null and b/forloop-tab-v2.png differ diff --git a/forloop-tab-v3.png b/forloop-tab-v3.png new file mode 100644 index 0000000000..997dd602a4 Binary files /dev/null and b/forloop-tab-v3.png differ diff --git a/frontend/BUGS.txt b/frontend/BUGS.txt new file mode 100644 index 0000000000..aae0762777 --- /dev/null +++ b/frontend/BUGS.txt @@ -0,0 +1,39 @@ +# Svelte 5 Migration - Bug Report +# Testing started: 2026-03-02 + +## Warnings (not blocking but worth fixing) + +1. [WARNING] binding_property_non_reactive in Grid.svelte:372:5 + - `bind:this={moveResizes[item.id]}` is binding to a non-reactive property + - File: src/lib/components/apps/svelte-grid/Grid.svelte + - Appears multiple times in App editor + - Status: NOT FIXED (non-blocking warning) + +2. [WARNING] legacy_recursive_reactive_block in RecomputeAllComponents.svelte + - Migrated `$:` reactive block that both accesses and updates the same reactive value + - File: src/lib/components/apps/editor/RecomputeAllComponents.svelte + - May cause recursive updates when converted to $effect + - Status: NOT FIXED (non-blocking warning) + +3. [WARNING] ownership_invalid_mutation in SchemaForm.svelte:70:16 + - Mutating unbound props (`schema`) is strongly discouraged + - Parent: src/lib/components/ApiConnectForm.svelte should use `bind:schema={...}` + - Appears when opening PostgreSQL resource creation form + - Status: NOT FIXED (non-blocking warning) + +4. [WARNING] ownership_invalid_binding in InputTransformSchemaForm.svelte + - Passes `schema` to InputTransformForm.svelte with `bind:`, but parent Pane.svelte didn't declare `schema` as binding + - Appears in flow editor when adding a TypeScript step + - Status: NOT FIXED (non-blocking warning) + +## Bugs + +1. [BUG] state_descriptors_fixed in Chart.svelte (Queue metrics drawer) + - Error: "Property descriptors defined on `$state` objects must contain `value` and always be `enumerable`, `configurable` and `writable`." + - Triggered by: Clicking "Queue metrics" on /workers page + - File: src/lib/components/chartjs-wrappers/Chart.svelte + - Root cause: Chart.js's `listenArrayEvents` calls Object.defineProperty on data arrays that are Svelte 5 $state proxies, which reject non-standard property descriptors + - Fix: Use $state.snapshot() to pass plain copies of data and options to Chart.js + - Status: FIXED + + diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c24e48e02d..647db391cd 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.649.0", + "version": "1.653.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.649.0", + "version": "1.653.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index d6c6554459..66e1da26b3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.649.0", + "version": "1.653.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index d8429a5ebf..8a79bd7109 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -15,6 +15,7 @@ import GitHubAppIntegration from './GitHubAppIntegration.svelte' import BedrockCredentialsCheck from './BedrockCredentialsCheck.svelte' import { isCloudHosted } from '$lib/cloud' + import ResourceGen from './copilot/ResourceGen.svelte' interface Props { resourceType: string @@ -149,6 +150,12 @@ }} class="as-json-toggle" /> + {#if resourceType == 'postgresql'} + import { run } from 'svelte/legacy' + import { createEventDispatcher } from 'svelte' import { Button, Drawer } from './common' import DrawerContent from './common/drawer/DrawerContent.svelte' @@ -6,24 +8,26 @@ import AppConnectInner from './AppConnectInner.svelte' import DarkModeObserver from './DarkModeObserver.svelte' - export let expressOAuthSetup = false - - let drawer: Drawer - let resourceType = '' - let step = 1 - let disabled = false - let isGoogleSignin = false - let manual = true - - let appConnectInner: AppConnectInner | undefined = undefined - - let rtToLoad: string | undefined = '' - export async function open(rt?: string) { - rtToLoad = rt - drawer.openDrawer?.() + interface Props { + expressOAuthSetup?: boolean } - $: appConnectInner && onRtToLoadChange(rtToLoad) + let { expressOAuthSetup = false }: Props = $props() + + let drawer: Drawer | undefined = $state() + let resourceType = $state('') + let step = $state(1) + let disabled = $state(false) + let isGoogleSignin = $state(false) + let manual = $state(true) + + let appConnectInner: AppConnectInner | undefined = $state(undefined) + + let rtToLoad: string | undefined = $state('') + export async function open(rt?: string) { + rtToLoad = rt + drawer?.openDrawer?.() + } function onRtToLoadChange(rtToLoad: string | undefined) { appConnectInner?.open(rtToLoad) @@ -31,7 +35,10 @@ const dispatch = createEventDispatcher() - let darkMode: boolean = false + let darkMode: boolean = $state(false) + run(() => { + appConnectInner && onRtToLoadChange(rtToLoad) + }) @@ -47,7 +54,7 @@ @@ -68,7 +75,7 @@ {/if} {#if isGoogleSignin} - diff --git a/frontend/src/lib/components/Description.svelte b/frontend/src/lib/components/Description.svelte index c3cd74c7d6..2078b74cd2 100644 --- a/frontend/src/lib/components/Description.svelte +++ b/frontend/src/lib/components/Description.svelte @@ -2,11 +2,19 @@ import { ExternalLink } from 'lucide-svelte' import { twMerge } from 'tailwind-merge' - export let link: string | undefined = undefined + + interface Props { + link?: string | undefined; + class?: string; + children?: import('svelte').Snippet; + } + + let { link = undefined, class: className = '', children }: Props = $props(); + -
- +
+ {@render children?.()} {#if link} Learn more initial) + if (untrackedInitial) { + if (untrackedInitial.type == 'script') { + replaceScript(untrackedInitial.script) + } else if (untrackedInitial.type == 'flow') { + replaceFlow(untrackedInitial.flow) } modeInitialized = true } @@ -601,9 +602,9 @@ } }) } - let token = $derived($page.url.searchParams.get('wm_token') ?? undefined) - let workspace = $derived($page.url.searchParams.get('workspace') ?? undefined) - let themeDarkRaw = $derived($page.url.searchParams.get('activeColorTheme')) + let token = $derived(page.url.searchParams.get('wm_token') ?? undefined) + let workspace = $derived(page.url.searchParams.get('workspace') ?? undefined) + let themeDarkRaw = $derived(page.url.searchParams.get('activeColorTheme')) let themeDark = $derived(themeDarkRaw == '2' || themeDarkRaw == '4') $effect.pre(() => { diff --git a/frontend/src/lib/components/DropdownSubmenuItem.svelte b/frontend/src/lib/components/DropdownSubmenuItem.svelte index 45346aaa3c..ed08fd7abd 100644 --- a/frontend/src/lib/components/DropdownSubmenuItem.svelte +++ b/frontend/src/lib/components/DropdownSubmenuItem.svelte @@ -1,4 +1,5 @@ diff --git a/frontend/src/lib/components/DropdownV2.svelte b/frontend/src/lib/components/DropdownV2.svelte index 902de661aa..c6243f3529 100644 --- a/frontend/src/lib/components/DropdownV2.svelte +++ b/frontend/src/lib/components/DropdownV2.svelte @@ -76,7 +76,7 @@ ids: { menu: dropdownId } } = createDropdownMenu({ positioning: { - placement + placement: untrack(() => placement) }, loop: true, onOpenChange: ({ next }) => { diff --git a/frontend/src/lib/components/DurationMs.svelte b/frontend/src/lib/components/DurationMs.svelte index 451925c0c6..60d2964ffe 100644 --- a/frontend/src/lib/components/DurationMs.svelte +++ b/frontend/src/lib/components/DurationMs.svelte @@ -4,9 +4,13 @@ import { Hourglass } from 'lucide-svelte' import WaitTimeWarning from './common/waitTimeWarning/WaitTimeWarning.svelte' - export let duration_ms: number - export let self_wait_time_ms: number | undefined = undefined - export let aggregate_wait_time_ms: number | undefined = undefined + interface Props { + duration_ms: number; + self_wait_time_ms?: number | undefined; + aggregate_wait_time_ms?: number | undefined; + } + + let { duration_ms, self_wait_time_ms = undefined, aggregate_wait_time_ms = undefined }: Props = $props();
diff --git a/frontend/src/lib/components/DynamicInput.svelte b/frontend/src/lib/components/DynamicInput.svelte index bd6cceaf99..692272d50e 100644 --- a/frontend/src/lib/components/DynamicInput.svelte +++ b/frontend/src/lib/components/DynamicInput.svelte @@ -113,10 +113,10 @@ } }) - let lastArgs = $state.snapshot(otherArgs) + let lastArgs = $state.snapshot(untrack(() => otherArgs)) let timeout: number | undefined = $state() - let nargs = $state($state.snapshot(otherArgs)) + let nargs = $state($state.snapshot(untrack(() => otherArgs))) $effect(() => { otherArgs untrack(() => clearTimeout(timeout)) diff --git a/frontend/src/lib/components/EditableSchemaForm.svelte b/frontend/src/lib/components/EditableSchemaForm.svelte index 99fe00bdb0..989ce6a2fe 100644 --- a/frontend/src/lib/components/EditableSchemaForm.svelte +++ b/frontend/src/lib/components/EditableSchemaForm.svelte @@ -286,7 +286,7 @@ } } - let jsonView: boolean = $state(customUi?.jsonOnly == true) + let jsonView: boolean = $state(untrack(() => customUi)?.jsonOnly == true) let schemaString: string = $state(JSON.stringify(schema, null, '\t')) let error: string | undefined = $state(undefined) let editor: SimpleEditor | undefined = $state(undefined) @@ -296,8 +296,8 @@ editor?.setCode(schemaString) } - const editTabDefaultSize = noPreview ? 100 : 50 - editPanelSize = editTab ? (editPanelInitialSize ?? editTabDefaultSize) : 0 + const editTabDefaultSize = untrack(() => noPreview) ? 100 : 50 + editPanelSize = untrack(() => editTab) ? (untrack(() => editPanelInitialSize) ?? editTabDefaultSize) : 0 let inputPanelSize = $state(100 - editPanelSize) let editPanelSizeSmooth = tweened(editPanelSize, { duration: 150 @@ -592,7 +592,7 @@ {argName} {#if !uiOnly}
- + {#snippet trigger()}
@@ -221,7 +224,7 @@
diff --git a/frontend/src/lib/components/FlowHistoryJobPicker.svelte b/frontend/src/lib/components/FlowHistoryJobPicker.svelte index 3a703d8bd7..26fa41f1af 100644 --- a/frontend/src/lib/components/FlowHistoryJobPicker.svelte +++ b/frontend/src/lib/components/FlowHistoryJobPicker.svelte @@ -12,13 +12,15 @@ selected?: string | undefined selectInitial?: boolean loading?: boolean + newFlow?: boolean } let { path, selected = undefined, selectInitial = false, - loading = $bindable(false) + loading = $bindable(false), + newFlow = false }: Props = $props() const dispatch = createEventDispatcher() @@ -41,7 +43,9 @@ } $effect(() => { - $workspaceStore && untrack(() => loadInitial()) + if ($workspaceStore && !newFlow) { + untrack(() => loadInitial()) + } }) diff --git a/frontend/src/lib/components/FlowInputViewer.svelte b/frontend/src/lib/components/FlowInputViewer.svelte index 94c25ffcc4..62aef2dd37 100644 --- a/frontend/src/lib/components/FlowInputViewer.svelte +++ b/frontend/src/lib/components/FlowInputViewer.svelte @@ -3,7 +3,11 @@ import FieldHeader from './FieldHeader.svelte' - export let schema: Schema | { [key: string]: unknown } | undefined + interface Props { + schema: Schema | { [key: string]: unknown } | undefined; + } + + let { schema }: Props = $props();
    diff --git a/frontend/src/lib/components/FlowLogViewerWrapper.svelte b/frontend/src/lib/components/FlowLogViewerWrapper.svelte index 4549fc8304..648dbc9f06 100644 --- a/frontend/src/lib/components/FlowLogViewerWrapper.svelte +++ b/frontend/src/lib/components/FlowLogViewerWrapper.svelte @@ -62,7 +62,7 @@ const timelineItems = $derived(timelineCompute?.items ?? undefined) const timelineNow = $derived(timelineCompute?.now ?? Date.now()) - let moduleTracker = new ChangeTracker($state.snapshot(job.raw_flow?.modules ?? [])) + let moduleTracker = new ChangeTracker($state.snapshot(untrack(() => job).raw_flow?.modules ?? [])) $effect(() => { readFieldsRecursively(job.raw_flow?.modules ?? []) untrack(() => moduleTracker.track($state.snapshot(job.raw_flow?.modules ?? []))) @@ -123,7 +123,7 @@ } let timelineAvailableWidths = $state>({}) - let lastJobId: string | undefined = $state(job.id) + let lastJobId: string | undefined = $state(untrack(() => job).id) const timelinelWidth = $derived.by(() => { const widths = Object.values(timelineAvailableWidths) diff --git a/frontend/src/lib/components/FlowPlugConnect.svelte b/frontend/src/lib/components/FlowPlugConnect.svelte index da70d882db..cd44f3d1bd 100644 --- a/frontend/src/lib/components/FlowPlugConnect.svelte +++ b/frontend/src/lib/components/FlowPlugConnect.svelte @@ -4,9 +4,13 @@ import AnimatedButton from './common/button/AnimatedButton.svelte' import { twMerge } from 'tailwind-merge' - export let connecting: boolean - export let id: undefined | string = undefined - export let wrapperClasses = '' + interface Props { + connecting: boolean; + id?: undefined | string; + wrapperClasses?: string; + } + + let { connecting, id = undefined, wrapperClasses = '' }: Props = $props(); diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index 28546536bb..f702b84397 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -588,6 +588,7 @@ {/if} { if (!currentJobId) { currentJobId = jobId @@ -607,7 +608,9 @@ />
- + {#if jobId} + + {/if} {#if job}
@@ -670,6 +673,10 @@
Loading history...
+ {:else} +
+ Flow status will display here +
{/if}
diff --git a/frontend/src/lib/components/FlowStatusViewer.svelte b/frontend/src/lib/components/FlowStatusViewer.svelte index 5da8b31419..950eb7354f 100644 --- a/frontend/src/lib/components/FlowStatusViewer.svelte +++ b/frontend/src/lib/components/FlowStatusViewer.svelte @@ -62,7 +62,7 @@ showLogsWithResult = false }: Props = $props() - let lastJobId: string = jobId + let lastJobId: string = untrack(() => jobId) let retryStatus = $state({ val: {} }) let globalRefreshes: Record Promise)[]> = $state({}) @@ -71,11 +71,11 @@ flowState, suspendStatus, retryStatus, - hideDownloadInGraph, - hideNodeDefinition, - hideTimeline, - hideJobId, - hideDownloadLogs + hideDownloadInGraph: untrack(() => hideDownloadInGraph), + hideNodeDefinition: untrack(() => hideNodeDefinition), + hideTimeline: untrack(() => hideTimeline), + hideJobId: untrack(() => hideJobId), + hideDownloadLogs: untrack(() => hideDownloadLogs) }) function loadOwner(path: string) { diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 410ba822ab..25731f82c8 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -181,7 +181,7 @@ let resultStreams: Record = $state({}) - if (onResultStreamUpdate == undefined) { + if (untrack(() => onResultStreamUpdate) == undefined) { onResultStreamUpdate = ({ jobId, result_stream @@ -234,7 +234,7 @@ }) let jobResults: any[] = $state( - flowJobIds?.flowJobs?.map((x, id) => `iter #${id + 1} not loaded by frontend yet`) ?? [] + untrack(() => flowJobIds)?.flowJobs?.map((x, id) => `iter #${id + 1} not loaded by frontend yet`) ?? [] ) let retry_selected = $state('') @@ -805,7 +805,7 @@ let destroyed = false - updateRecursiveRefresh(jobId) + updateRecursiveRefresh(untrack(() => jobId)) async function updateJobId() { if (jobId !== job?.id || innerModules == undefined) { diff --git a/frontend/src/lib/components/FlowViewer.svelte b/frontend/src/lib/components/FlowViewer.svelte index 8a78637519..2355ddf68d 100644 --- a/frontend/src/lib/components/FlowViewer.svelte +++ b/frontend/src/lib/components/FlowViewer.svelte @@ -1,4 +1,5 @@ {#if tabular} diff --git a/frontend/src/lib/components/GraphqlSchemaViewer.svelte b/frontend/src/lib/components/GraphqlSchemaViewer.svelte index 7929083d9a..1ab4c68f00 100644 --- a/frontend/src/lib/components/GraphqlSchemaViewer.svelte +++ b/frontend/src/lib/components/GraphqlSchemaViewer.svelte @@ -5,10 +5,17 @@ import { onDestroy, onMount } from 'svelte' - let divEl: HTMLDivElement | null = null + let divEl: HTMLDivElement | null = $state(null) let editor: meditor.IStandaloneCodeEditor - export let code: string = '' + + interface Props { + code?: string; + class?: string; + } + + let { code = '', class: className = '' }: Props = $props(); + async function loadMonaco() { editor = meditor.create(divEl as HTMLDivElement, { @@ -43,4 +50,4 @@ }) -
+
diff --git a/frontend/src/lib/components/GroupEditor.svelte b/frontend/src/lib/components/GroupEditor.svelte index a8391bd8a6..5a8c4cb411 100644 --- a/frontend/src/lib/components/GroupEditor.svelte +++ b/frontend/src/lib/components/GroupEditor.svelte @@ -159,12 +159,14 @@ {/if} {#if members} - - - user - - - + + {#snippet headerRow()} + + user + + + + {/snippet} {#snippet body()} {#each members ?? [] as { member_name, role }} @@ -301,10 +303,12 @@ {#if instance_group?.emails}

Members from the instance group

- - - user - + + {#snippet headerRow()} + + user + + {/snippet} {#snippet body()} {#each instance_group?.emails ?? [] as email} diff --git a/frontend/src/lib/components/IdEditorInput.svelte b/frontend/src/lib/components/IdEditorInput.svelte index 7b85d8e52b..611dbc2d3e 100644 --- a/frontend/src/lib/components/IdEditorInput.svelte +++ b/frontend/src/lib/components/IdEditorInput.svelte @@ -1,4 +1,5 @@ {#if entries.length} diff --git a/frontend/src/lib/components/InstanceGroupEditor.svelte b/frontend/src/lib/components/InstanceGroupEditor.svelte index cf2a3ded0a..f3d8df33de 100644 --- a/frontend/src/lib/components/InstanceGroupEditor.svelte +++ b/frontend/src/lib/components/InstanceGroupEditor.svelte @@ -1,4 +1,6 @@
@@ -85,17 +91,20 @@
{#if members} - - user - - - - {#each members as { member_email }} - {member_email} - - - - {/each} - + > + + {/each} + + {/snippet} {:else}
diff --git a/frontend/src/lib/components/InstanceNameEditor.svelte b/frontend/src/lib/components/InstanceNameEditor.svelte index 31fc3d738b..cb710cddce 100644 --- a/frontend/src/lib/components/InstanceNameEditor.svelte +++ b/frontend/src/lib/components/InstanceNameEditor.svelte @@ -1,4 +1,7 @@
- +
+ {/snippet} diff --git a/frontend/src/lib/components/PageHeader.svelte b/frontend/src/lib/components/PageHeader.svelte index 07b02227d4..556ac1b0cc 100644 --- a/frontend/src/lib/components/PageHeader.svelte +++ b/frontend/src/lib/components/PageHeader.svelte @@ -1,11 +1,23 @@
@@ -31,9 +43,9 @@ {/if} - {#if $$slots.default} + {#if children}
- + {@render children?.()}
{/if}
diff --git a/frontend/src/lib/components/ParqetCsvTableRenderer.svelte b/frontend/src/lib/components/ParqetCsvTableRenderer.svelte index 07a8c267a9..262be32363 100644 --- a/frontend/src/lib/components/ParqetCsvTableRenderer.svelte +++ b/frontend/src/lib/components/ParqetCsvTableRenderer.svelte @@ -1,4 +1,6 @@ @@ -182,7 +189,7 @@
- mountGrid()}> diff --git a/frontend/src/lib/components/PermissionHistory.svelte b/frontend/src/lib/components/PermissionHistory.svelte index 82d646bd53..23f3ec151f 100644 --- a/frontend/src/lib/components/PermissionHistory.svelte +++ b/frontend/src/lib/components/PermissionHistory.svelte @@ -79,12 +79,14 @@

No permission changes recorded yet

{:else} - - Changed By - Change Type - Affected - Date - + {#snippet headerRow()} + + Changed By + Change Type + Affected + Date + + {/snippet} {#snippet body()} {#each history as change} diff --git a/frontend/src/lib/components/PersistentScriptDrawer.svelte b/frontend/src/lib/components/PersistentScriptDrawer.svelte index aa5c869f31..23fa122b7d 100644 --- a/frontend/src/lib/components/PersistentScriptDrawer.svelte +++ b/frontend/src/lib/components/PersistentScriptDrawer.svelte @@ -10,19 +10,19 @@ import { Hourglass, Loader2, Play, RefreshCw } from 'lucide-svelte' let dispatch = createEventDispatcher() - let drawer: Drawer + let drawer: Drawer | undefined = $state() - let script: Script - let loadQueuedJobs = true - let queuedJobsLoading = false + let script: Script | undefined = $state() + let loadQueuedJobs = $state(true) + let queuedJobsLoading = $state(false) let queuedJobs: { status: 'running' | 'queued' jobId: string scheduledFor: string scriptHash: string - }[] = [] + }[] = $state([]) - let cancellingInProgress = false + let cancellingInProgress = $state(false) async function continuouslyLoadQueuedJobs() { while (loadQueuedJobs) { @@ -40,7 +40,7 @@ let qjs = await JobService.listQueue({ workspace: $workspaceStore ?? '', orderDesc: false, - scriptPathExact: script.path + scriptPathExact: script?.path }) let loadingQueuedJobs: { status: 'running' | 'queued' @@ -71,12 +71,12 @@ cancellingInProgress = true await JobService.cancelPersistentQueuedJobs({ workspace: $workspaceStore ?? '', - path: script.path, + path: script?.path ?? '', requestBody: { reason: undefined } }) - sendUserToast(`All jobs cancelled for ${script.path}`) + sendUserToast(`All jobs cancelled for ${script?.path}`) cancellingInProgress = false } @@ -88,12 +88,12 @@ script = persistentScript! loadQueuedJobs = true continuouslyLoadQueuedJobs() - drawer.openDrawer?.() + drawer?.openDrawer?.() } async function exit() { loadQueuedJobs = false - drawer.closeDrawer?.() + drawer?.closeDrawer?.() } onDestroy(() => { @@ -117,51 +117,57 @@ >

- Queued jobs for {script.path} + Queued jobs for {script?.path}

- - Script Hash - Job ID - Status - Scheduled For - - - {#each queuedJobs as { jobId, status, scriptHash, scheduledFor }} - - - - {scriptHash} - - - - {jobId.substring(24)} - - - {#if status === 'running'} - - - - {:else} - - - - {/if} - - {scheduledFor} - - {/each} - + {#snippet headerRow()} + + Script Hash + Job ID + Status + Scheduled For + + {/snippet} + {#snippet body()} + + {#each queuedJobs as { jobId, status, scriptHash, scheduledFor }} + + + + {scriptHash} + + + + {jobId.substring(24)} + + + {#if status === 'running'} + + + + {:else} + + + + {/if} + + {scheduledFor} + + {/each} + + {/snippet} {#snippet actions()} diff --git a/frontend/src/lib/components/Popover.svelte b/frontend/src/lib/components/Popover.svelte index 1fa0a0d88d..07447b2e4a 100644 --- a/frontend/src/lib/components/Popover.svelte +++ b/frontend/src/lib/components/Popover.svelte @@ -43,10 +43,10 @@ onClick }: Props = $props() - const [popperRef, popperContent] = createPopperActions({ placement }) + const [popperRef, popperContent] = createPopperActions({ placement: untrack(() => placement) }) const popperOptions: PopperOptions<{}> = { - placement, + placement: untrack(() => placement), strategy: 'fixed', modifiers: [ { name: 'offset', options: { offset: [8, 8] } }, diff --git a/frontend/src/lib/components/PrefixedInput.svelte b/frontend/src/lib/components/PrefixedInput.svelte index c548418855..2a1dccf386 100644 --- a/frontend/src/lib/components/PrefixedInput.svelte +++ b/frontend/src/lib/components/PrefixedInput.svelte @@ -1,4 +1,5 @@ diff --git a/frontend/src/lib/components/RadioButton.svelte b/frontend/src/lib/components/RadioButton.svelte index efa9a62b40..c7e5dc8dce 100644 --- a/frontend/src/lib/components/RadioButton.svelte +++ b/frontend/src/lib/components/RadioButton.svelte @@ -1,13 +1,24 @@ @@ -28,7 +39,7 @@ class="sr-only" bind:group={value} aria-labelledby="memory-option-0-label" - on:click={() => dispatch('change', val)} + onclick={() => dispatch('change', val)} />

{#if typeof label !== 'string'} diff --git a/frontend/src/lib/components/Range.svelte b/frontend/src/lib/components/Range.svelte index 0e92ac72de..72129aada2 100644 --- a/frontend/src/lib/components/Range.svelte +++ b/frontend/src/lib/components/Range.svelte @@ -1,29 +1,47 @@

- +
{#if max <= min}
Impossible to display range: {`max (${max}) <= min (${min})`}
+ import { untrack } from 'svelte' import { GitSyncService } from '$lib/gen' import Select from './select/Select.svelte' @@ -32,7 +33,7 @@ }: Props = $props() // Track all loaded repositories across pages - let loadedRepositories = $state(initialRepositories) + let loadedRepositories = $state(untrack(() => initialRepositories)) let currentPage = $state(1) let isLoadingMore = $state(false) diff --git a/frontend/src/lib/components/Required.svelte b/frontend/src/lib/components/Required.svelte index 2163efa976..3ee1deec1a 100644 --- a/frontend/src/lib/components/Required.svelte +++ b/frontend/src/lib/components/Required.svelte @@ -1,12 +1,19 @@ {#if required} - * + * {:else if detail || detail != ''} - ({detail != '' ? `${detail}` : ''}) {/if} diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 5a5943390e..70aaf6a2ee 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -24,12 +24,12 @@ import GitHubAppIntegration from './GitHubAppIntegration.svelte' import Button from './common/button/Button.svelte' import { clearJsonSchemaResourceCache } from './schema/jsonSchemaResource.svelte' + import ResourceGen from './copilot/ResourceGen.svelte' interface Props { canSave?: boolean resource_type?: string | undefined path?: string - newResource?: boolean hidePath?: boolean onChange?: (args: { path: string; args: Record; description: string }) => void defaultValues?: Record | undefined @@ -39,7 +39,6 @@ canSave = $bindable(true), resource_type = $bindable(undefined), path = $bindable(''), - newResource = false, hidePath = false, onChange, defaultValues = undefined @@ -62,6 +61,7 @@ let resourceTypeInfo: ResourceType | undefined = $state(undefined) let editDescription = $state(false) let viewJsonSchema = $state(false) + let newResource = $derived(!path) const dispatch = createEventDispatcher() @@ -81,7 +81,7 @@ .map(([k, _]) => k) } - if (!newResource) { + if (!untrack(() => newResource)) { initEdit() } else if (resource_type) { loadResourceType() @@ -270,6 +270,13 @@ right: 'As JSON' }} /> + {#if resourceToEdit?.resource_type === 'nats' || resourceToEdit?.resource_type === 'kafka'} {:else} @@ -296,9 +303,17 @@ {#if loadingSchema} {:else if !viewJsonSchema && resourceTypeInfo?.is_fileset} -
- Fileset -
+
+
Fileset
+ +
{:else if !viewJsonSchema && resourceSchema && resourceSchema?.properties} {#if resourceTypeInfo?.format_extension} diff --git a/frontend/src/lib/components/ResourceEditorDrawer.svelte b/frontend/src/lib/components/ResourceEditorDrawer.svelte index 50db916f13..dae6943868 100644 --- a/frontend/src/lib/components/ResourceEditorDrawer.svelte +++ b/frontend/src/lib/components/ResourceEditorDrawer.svelte @@ -5,50 +5,44 @@ import { Loader2, Save } from 'lucide-svelte' - let drawer: Drawer - let canSave = true - let resource_type: string | undefined = undefined - let defaultValues: Record | undefined = undefined + let drawer: Drawer | undefined = $state() + let canSave = $state(true) + let resource_type: string | undefined = $state(undefined) + let defaultValues: Record | undefined = $state(undefined) let resourceEditor: { editResource: () => void; createResource: () => void } | undefined = - undefined + $state(undefined) - let path: string | undefined = undefined + let path: string | undefined = $state(undefined) - let newResource = false export async function initEdit(p: string): Promise { resource_type = undefined - newResource = false path = p - drawer.openDrawer?.() + drawer?.openDrawer?.() } export async function initNew( resourceType: string, nDefaultValues?: Record ): Promise { - newResource = true path = undefined resource_type = resourceType defaultValues = nDefaultValues - drawer.openDrawer?.() + drawer?.openDrawer?.() } - let mode: 'edit' | 'new' = newResource ? 'new' : 'edit' - - $: path ? (mode = 'edit') : (mode = 'new') + let mode: 'edit' | 'new' = $derived(!path ? 'new' : 'edit') {#await import('./ResourceEditor.svelte')} {:then Module} diff --git a/frontend/src/lib/components/ResourcePicker.svelte b/frontend/src/lib/components/ResourcePicker.svelte index 6806e60d3c..fddf9205f5 100644 --- a/frontend/src/lib/components/ResourcePicker.svelte +++ b/frontend/src/lib/components/ResourcePicker.svelte @@ -152,7 +152,7 @@ loading = false } - let previousResourceType = resourceType + let previousResourceType = untrack(() => resourceType) $effect(() => { $workspaceStore && resourceType diff --git a/frontend/src/lib/components/RunForm.svelte b/frontend/src/lib/components/RunForm.svelte index 969576a06b..da0c87a15f 100644 --- a/frontend/src/lib/components/RunForm.svelte +++ b/frontend/src/lib/components/RunForm.svelte @@ -15,7 +15,7 @@ import Popover from './meltComponents/Popover.svelte' import { Calendar, Check, CornerDownLeft } from 'lucide-svelte' import RunFormAdvancedPopup from './RunFormAdvancedPopup.svelte' - import { page } from '$app/stores' + import { page } from '$app/state' import { replaceState } from '$app/navigation' import JsonInputs from '$lib/components/JsonInputs.svelte' import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte' @@ -108,7 +108,7 @@ nurl.hash = computeSharableHash(args) try { - replaceState(nurl.toString(), $page.state) + replaceState(nurl.toString(), page.state) } catch (e) { console.error(e) } diff --git a/frontend/src/lib/components/RunFormAdvancedPopup.svelte b/frontend/src/lib/components/RunFormAdvancedPopup.svelte index 7d6ef01bb2..399e55f772 100644 --- a/frontend/src/lib/components/RunFormAdvancedPopup.svelte +++ b/frontend/src/lib/components/RunFormAdvancedPopup.svelte @@ -8,7 +8,9 @@ import { WorkerService } from '$lib/gen' import DateTimeInput from './DateTimeInput.svelte' - export let runnable: + + interface Props { + runnable: | { summary?: string description?: string @@ -21,11 +23,18 @@ created_by?: string extra_perms?: Record } - | undefined + | undefined; + scheduledForStr: string | undefined; + invisible_to_owner: boolean | undefined; + overrideTag: string | undefined; + } - export let scheduledForStr: string | undefined - export let invisible_to_owner: boolean | undefined - export let overrideTag: string | undefined + let { + runnable, + scheduledForStr = $bindable(), + invisible_to_owner = $bindable(), + overrideTag = $bindable() + }: Props = $props(); loadWorkerGroups() async function loadWorkerGroups() { diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte index d03495c052..5aee359a4a 100644 --- a/frontend/src/lib/components/RunsPage.svelte +++ b/frontend/src/lib/components/RunsPage.svelte @@ -96,8 +96,8 @@ let batchRerunOptionsIsOpen = $state(false) // Initialize path filter from route param if provided and not already set via query params - if (initialPath && !filters.val.path) { - filters.val.path = initialPath + if (untrack(() => initialPath) && !filters.val.path) { + filters.val.path = untrack(() => initialPath) } // Apply persistent toggle values from local storage if URL doesn't specify them diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index a07ea85d25..55bb25c5ef 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -13,6 +13,7 @@ interface Props { fromWorkspaceSettings?: boolean readOnlyMode: boolean + allowDelete?: boolean initialFileKey?: { s3: string; storage?: string } | undefined selectedFileKey?: { s3: string; storage?: string } | undefined folderOnly?: boolean @@ -24,6 +25,7 @@ let { fromWorkspaceSettings = false, readOnlyMode, + allowDelete = false, initialFileKey = $bindable(undefined), selectedFileKey = $bindable(undefined), folderOnly = false, @@ -94,6 +96,7 @@ }} {fromWorkspaceSettings} {readOnlyMode} + {allowDelete} bind:initialFileKey bind:selectedFileKey bind:workspaceSettingsInitialized diff --git a/frontend/src/lib/components/S3FilePickerInner.svelte b/frontend/src/lib/components/S3FilePickerInner.svelte index 380e3d9a35..74b85bd327 100644 --- a/frontend/src/lib/components/S3FilePickerInner.svelte +++ b/frontend/src/lib/components/S3FilePickerInner.svelte @@ -81,6 +81,7 @@ count: number } > + allowDelete?: boolean replaceUnauthorizedWarning?: Snippet listStoredFilesRequest?: (d: ListStoredFilesData) => CancelablePromise loadFilePreviewRequest?: (d: LoadFilePreviewData) => CancelablePromise @@ -102,11 +103,12 @@ folderOnly = false, regexFilter = undefined, hideS3SpecificDetails = false, - rootPath = '', + rootPath: initialRootPath = '', workspaceSettingsInitialized = $bindable(true), storage = $bindable(undefined), uploadModalOpen = $bindable(false), allFilesByKey = $bindable({}), + allowDelete = false, replaceUnauthorizedWarning, listStoredFilesRequest = HelpersService.listStoredFiles, loadFilePreviewRequest = HelpersService.loadFilePreview, @@ -116,6 +118,7 @@ testConnectionRequest = HelpersService.datasetStorageTestConnection }: Props = $props() + let rootPath = $state(initialRootPath) let rootPathNestingLevel = $derived(1 * (rootPath.split('/').length - 1)) let csvSeparatorChar: string = $state(',') @@ -263,7 +266,7 @@ } } } - displayedFileKeys = displayedFileKeys.sort() + displayedFileKeys = [...new Set(displayedFileKeys)].sort() fileListLoading = false fileInfoLoading = false } @@ -381,7 +384,7 @@ } } } - displayedFileKeys = displayedFileKeys.sort() + displayedFileKeys = [...new Set(displayedFileKeys)].sort() } async function clearAndLoadFiles({ keepFilter }: { keepFilter?: boolean } = {}) { @@ -424,9 +427,16 @@ export async function open(_preSelectedFileKey: S3Object | undefined = undefined) { const preSelectedFileKey = _preSelectedFileKey && parseS3Object(_preSelectedFileKey) storage = preSelectedFileKey?.storage - if (preSelectedFileKey !== undefined) { + if (preSelectedFileKey !== undefined && preSelectedFileKey.s3.endsWith('/')) { + rootPath = preSelectedFileKey.s3 + filter = '' + selectedFileKey = undefined + } else if (preSelectedFileKey !== undefined) { + rootPath = '' initialFileKey = { ...preSelectedFileKey } selectedFileKey = { ...preSelectedFileKey } + } else { + rootPath = '' } reloadContent() } @@ -461,7 +471,7 @@ if (selectedFileKey !== undefined) { if (allFilesByKey[selectedFileKey.s3] === undefined) { selectedFileKey = { s3: '', storage } - } else { + } else if (allFilesByKey[selectedFileKey.s3].type !== 'folder') { loadFileMetadataPlusPreviewAsync(selectedFileKey.s3) } } @@ -518,7 +528,7 @@ } } } - displayedFileKeys = displayedFileKeys.sort() + displayedFileKeys = [...new Set(displayedFileKeys)].sort() } else { selectedFileKey = { s3: item_key, @@ -719,8 +729,10 @@ startIcon={{ icon: MoveRight }} iconOnly={true} /> + {/if} + {#if !readOnlyMode || allowDelete} +
{#if customUi?.settingsPanel?.metadata?.disableScriptKind !== true}
{#snippet header()} @@ -1652,8 +1678,10 @@ /> {:else if script.on_behalf_of_email && !canPreserve} - Currently: {originalOnBehalfOfEmail ?? script.on_behalf_of_email}. - Will be set to {$userStore?.email} on deploy (requires admin or wm_deployers group to override) + Currently: {originalOnBehalfOfEmail ?? script.on_behalf_of_email}. Will be set to {$userStore?.email} on + deploy (requires admin or wm_deployers group to override) {/if} diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index 5fe03b03fd..f310010c18 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -100,7 +100,7 @@ path: string | undefined lang: Preview['language'] kind?: string | undefined - template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' + template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' | 'claudesandbox' tag: string | undefined initialArgs?: Record fixedOverflowWidgets?: boolean @@ -123,7 +123,7 @@ lastDeployedCode?: string | undefined disableAi?: boolean assets?: AssetWithAltAccessType[] - editor_bar_right?: import('svelte').Snippet + editorBarRight?: import('svelte').Snippet enablePreprocessorSnippet?: boolean } @@ -155,7 +155,7 @@ lastDeployedCode = undefined, disableAi = false, assets = $bindable(), - editor_bar_right, + editorBarRight, enablePreprocessorSnippet = false }: Props = $props() @@ -883,7 +883,7 @@ } } - setContext('disableTooltips', customUi?.disableTooltips === true) + setContext('disableTooltips', untrack(() => customUi)?.disableTooltips === true) let codePanelSize = $state(70) let testPanelSize = $state(30) @@ -1042,7 +1042,7 @@ bind:showHistoryDrawer > {#snippet right()} - {@render editor_bar_right?.()} + {@render editorBarRight?.()} {/snippet} {/if} diff --git a/frontend/src/lib/components/ScriptPicker.svelte b/frontend/src/lib/components/ScriptPicker.svelte index 5fa0618edd..803684f49e 100644 --- a/frontend/src/lib/components/ScriptPicker.svelte +++ b/frontend/src/lib/components/ScriptPicker.svelte @@ -52,7 +52,7 @@ let lang: SupportedLanguage | undefined = $state() let options: [[string, any, any, string | undefined]] = [['Script', 'script', Code2, undefined]] - allowFlow && options.push(['Flow', 'flow', FlowIcon, '#14b8a6']) + untrack(() => allowFlow) && options.push(['Flow', 'flow', FlowIcon, '#14b8a6']) const dispatch = createEventDispatcher() async function loadItems(): Promise { diff --git a/frontend/src/lib/components/ScriptWrapper.svelte b/frontend/src/lib/components/ScriptWrapper.svelte index 5e53dc09f3..eec781ce77 100644 --- a/frontend/src/lib/components/ScriptWrapper.svelte +++ b/frontend/src/lib/components/ScriptWrapper.svelte @@ -1,11 +1,12 @@ diff --git a/frontend/src/lib/components/Scrollable.svelte b/frontend/src/lib/components/Scrollable.svelte index 37e3ae2933..7c34b3cbe1 100644 --- a/frontend/src/lib/components/Scrollable.svelte +++ b/frontend/src/lib/components/Scrollable.svelte @@ -2,14 +2,19 @@ import { onMount, onDestroy } from 'svelte' import { twMerge } from 'tailwind-merge' - let isAtBottom: boolean = false - let isScrollable = false + let isAtBottom: boolean = $state(false) + let isScrollable = $state(false) - export let id: string | null | undefined = undefined - export let scrollableClass: string = '' - export let shiftedShadow: boolean = false + interface Props { + id?: string | null | undefined + scrollableClass?: string + shiftedShadow?: boolean + children?: import('svelte').Snippet + } + + let { id = undefined, scrollableClass = '', shiftedShadow = false, children }: Props = $props() let mutationObserver: MutationObserver - let el: HTMLDivElement + let el: HTMLDivElement | undefined = $state() function handleScroll(event) { const scrollableElement = event.target @@ -20,7 +25,8 @@ } function checkIfScrollable(el) { - return el.scrollHeight > el.clientHeight + if (!el) return false + return el?.scrollHeight > el?.clientHeight } function observeScrollability(el) { @@ -33,7 +39,7 @@ } export function scrollIntoView(top: number) { - el.scrollTo({ top, behavior: 'smooth' }) + el?.scrollTo({ top, behavior: 'smooth' }) } onMount(() => { observeScrollability(el) @@ -45,8 +51,8 @@
-
- +
+ {@render children?.()}
{#if !isAtBottom && isScrollable}
opts)) function filterItems() { let trimmed = filter.trim() diff --git a/frontend/src/lib/components/ServiceLogsInner.svelte b/frontend/src/lib/components/ServiceLogsInner.svelte index 0f4f923cd6..df0298f579 100644 --- a/frontend/src/lib/components/ServiceLogsInner.svelte +++ b/frontend/src/lib/components/ServiceLogsInner.svelte @@ -22,7 +22,7 @@ import SplitPanesOrColumnOnMobile from './splitPanes/SplitPanesOrColumnOnMobile.svelte' import Select from './select/Select.svelte' import { goto } from '$lib/navigation' - import { page } from '$app/stores' + import { page } from '$app/state' import { watch } from 'runed' interface Props { @@ -169,13 +169,13 @@ type Selected = { mode: string; workerGroup: string; hostname: string } let initialSelected = - $page.url.searchParams.get('mode') && - $page.url.searchParams.get('workerGroup') && - $page.url.searchParams.get('hostname') + page.url.searchParams.get('mode') && + page.url.searchParams.get('workerGroup') && + page.url.searchParams.get('hostname') ? { - mode: $page.url.searchParams.get('mode')!, - workerGroup: $page.url.searchParams.get('workerGroup')!, - hostname: $page.url.searchParams.get('hostname')! + mode: page.url.searchParams.get('mode')!, + workerGroup: page.url.searchParams.get('workerGroup')!, + hostname: page.url.searchParams.get('hostname')! } : undefined let selected: Selected | undefined = $state(initialSelected) @@ -523,7 +523,13 @@ {#if allLogs == undefined}
{:else if Object.keys(allLogs).length == 0} -
No logs
+
+ No logs + Search only covers a recent time window, configurable in instance settings + under Indexer. +
{:else if minTs && maxTs} {@const minTsN = new Date(minTs).getTime()} {@const maxTsN = new Date(maxTs).getTime()} @@ -663,7 +669,7 @@ { + onClick={() => { let logLineNumber = document.line_number[0] let logFile = document.file_name[0] let host = document.host[0] diff --git a/frontend/src/lib/components/ShareModal.svelte b/frontend/src/lib/components/ShareModal.svelte index e5ad54f88b..10cf5f09e0 100644 --- a/frontend/src/lib/components/ShareModal.svelte +++ b/frontend/src/lib/components/ShareModal.svelte @@ -35,6 +35,7 @@ | 'postgres_trigger' | 'gcp_trigger' | 'email_trigger' + | 'volume' let kind: Kind let path: string = $state('') @@ -53,13 +54,17 @@ let drawer: Drawer | undefined = $state() let own = $state(false) - export async function openDrawer(newPath: string, kind_l: Kind) { + export async function openDrawer(newPath: string, kind_l: Kind, isOwnerOverride?: boolean) { path = newPath kind = kind_l loadAcls() loadGroups() loadUsernames() - loadOwner() + if (isOwnerOverride !== undefined) { + own = isOwnerOverride + } else { + loadOwner() + } drawer?.openDrawer() } @@ -154,12 +159,14 @@ {/if} {#if acls?.length > 0} - - - owner - - - + + {#snippet headerRow()} + + owner + + + + {/snippet} {#snippet body()} {#each acls as [owner, write]} diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index 2fdd14e3f9..31da7fd802 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -130,7 +130,7 @@ const dispatch = createEventDispatcher() - const uri = `file:///${hash}.${langToExt(lang)}` + const uri = `file:///${untrack(() => hash)}.${langToExt(untrack(() => lang))}` export function getCode(): string { if (valueAfterDispose != undefined) { diff --git a/frontend/src/lib/components/Slider.svelte b/frontend/src/lib/components/Slider.svelte index c2ca0abbdd..e3269cde09 100644 --- a/frontend/src/lib/components/Slider.svelte +++ b/frontend/src/lib/components/Slider.svelte @@ -5,10 +5,21 @@ import Tooltip from './Tooltip.svelte' import { twMerge } from 'tailwind-merge' - export let text: string - export let tooltip: string | undefined = undefined - export let view = false - export let size: 'xs' | 'sm' | 'md' | 'lg' = 'md' + interface Props { + text: string; + tooltip?: string | undefined; + view?: boolean; + size?: 'xs' | 'sm' | 'md' | 'lg'; + children?: import('svelte').Snippet; + } + + let { + text, + tooltip = undefined, + view = $bindable(false), + size = 'md', + children + }: Props = $props();
- - {#if isCloudHosted()} - The cloud version is updated daily. - {:else} - How to update?
- - docker: docker compose up -d
- - helm - {/if} - + {#snippet text()} + + {#if isCloudHosted()} + The cloud version is updated daily. + {:else} + How to update?
+ - docker: docker compose up -d
+ - helm + {/if} + + {/snippet} {/snippet} {#snippet actions()} diff --git a/frontend/src/lib/components/SuperadminSettingsInner.svelte b/frontend/src/lib/components/SuperadminSettingsInner.svelte index 4d6b0e33f1..1f33e768d8 100644 --- a/frontend/src/lib/components/SuperadminSettingsInner.svelte +++ b/frontend/src/lib/components/SuperadminSettingsInner.svelte @@ -8,7 +8,7 @@ import { sendUserToast } from '$lib/toast' import { base } from '$lib/base' import SearchItems from './SearchItems.svelte' - import { page } from '$app/stores' + import { page } from '$app/state' import { goto as gotoUrl } from '$app/navigation' import Version from './Version.svelte' import Uptodate from './Uptodate.svelte' @@ -50,9 +50,9 @@ } = $props() function removeHash() { - const index = $page.url.href.lastIndexOf('#') + const index = page.url.href.lastIndexOf('#') if (index === -1) return - const hashRemoved = $page.url.href.slice(0, index) + const hashRemoved = page.url.href.slice(0, index) gotoUrl(hashRemoved) } diff --git a/frontend/src/lib/components/TableCustom.svelte b/frontend/src/lib/components/TableCustom.svelte index e40a2c9706..a508d48c8f 100644 --- a/frontend/src/lib/components/TableCustom.svelte +++ b/frontend/src/lib/components/TableCustom.svelte @@ -1,37 +1,53 @@ -
+
- + {@render headerRow?.()} - + {@render body?.()}
{#if paginated}
diff --git a/frontend/src/lib/components/TemplateEditor.svelte b/frontend/src/lib/components/TemplateEditor.svelte index f3ad00ffc3..98f11c3076 100644 --- a/frontend/src/lib/components/TemplateEditor.svelte +++ b/frontend/src/lib/components/TemplateEditor.svelte @@ -412,7 +412,7 @@ const lang = 'template' const dispatch = createEventDispatcher() - const uri = `file:///${hash}.ts` + const uri = `file:///${untrack(() => hash)}.ts` export function insertAtCursor(code: string): void { if (editor) { @@ -438,7 +438,6 @@ let cip let extraModel - let width = $state(0) // let widgets: HTMLElement | undefined = document.getElementById('monaco-widgets-root') ?? undefined let initialized = $state(false) @@ -545,9 +544,6 @@ if (divEl) { divEl.style.height = `${contentHeight}px` } - try { - editor?.layout({ width, height: contentHeight }) - } catch {} } editor.onDidContentSizeChange(updateHeight) updateHeight() @@ -718,7 +714,6 @@ bind:this={divEl} style="height: 18px;" class="template nonmain-editor rounded-md overflow-clip {!editor ? 'hidden' : ''}" - bind:clientWidth={width} >
diff --git a/frontend/src/lib/components/TestConnection.svelte b/frontend/src/lib/components/TestConnection.svelte index 576bbc9334..7c59a2a324 100644 --- a/frontend/src/lib/components/TestConnection.svelte +++ b/frontend/src/lib/components/TestConnection.svelte @@ -7,10 +7,19 @@ import { workspaceStore } from '$lib/stores' import { tryEvery } from '$lib/utils' - export let workspaceOverride: string | undefined = undefined - export let resourceType: string | undefined - export let args: Record | any = {} - export let buttonTextOverride: string | undefined = undefined + interface Props { + workspaceOverride?: string | undefined; + resourceType: string | undefined; + args?: Record | any; + buttonTextOverride?: string | undefined; + } + + let { + workspaceOverride = undefined, + resourceType, + args = {}, + buttonTextOverride = undefined + }: Props = $props(); const scripts: { [key: string]: { @@ -166,7 +175,7 @@ export async function main(bucket: any) { } } - let loading = false + let loading = $state(false) async function testConnection() { if (!resourceType) return loading = true diff --git a/frontend/src/lib/components/TimelineBar.svelte b/frontend/src/lib/components/TimelineBar.svelte index 7c4e170d93..e4fb511450 100644 --- a/frontend/src/lib/components/TimelineBar.svelte +++ b/frontend/src/lib/components/TimelineBar.svelte @@ -35,7 +35,7 @@ {/if} {/snippet} {#if len > 0} - {#if len}{msToSec(len, 1)}s{/if} {/if} diff --git a/frontend/src/lib/components/Toast.svelte b/frontend/src/lib/components/Toast.svelte index 13e857ed6f..034887235e 100644 --- a/frontend/src/lib/components/Toast.svelte +++ b/frontend/src/lib/components/Toast.svelte @@ -82,14 +82,14 @@ } }) - let color = classes[type] + let color = classes[untrack(() => type)] let containerClass = { success: 'toast-success', error: 'toast-error', info: 'toast-info', warning: 'toast-warning' - }[type] + }[untrack(() => type)] let Icon = $derived(icons[type]) diff --git a/frontend/src/lib/components/Toggle.svelte b/frontend/src/lib/components/Toggle.svelte index 6b502bcbfa..4ae37d325f 100644 --- a/frontend/src/lib/components/Toggle.svelte +++ b/frontend/src/lib/components/Toggle.svelte @@ -3,7 +3,7 @@ const bubble = createBubbler() import { classNames } from '$lib/utils' - import { createEventDispatcher } from 'svelte' + import { createEventDispatcher, untrack } from 'svelte' import { twMerge } from 'tailwind-merge' import Tooltip from './Tooltip.svelte' import { AlertTriangle } from 'lucide-svelte' @@ -54,7 +54,7 @@ }: Props = $props() const dispatch = createEventDispatcher<{ change: boolean }>() - const bothOptions = Boolean(options.left) && Boolean(options.right) + const bothOptions = Boolean(untrack(() => options).left) && Boolean(untrack(() => options).right)